VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAll.cpp@ 108156

Last change on this file since 108156 was 108156, checked in by vboxsync, 11 days ago

VMM/PGM: Try keep the x86-only stuff more together. jiraref:VBP-1531
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 157.6 KB
Line 
1/* $Id: PGMAll.cpp 108156 2025-02-11 12:26:07Z vboxsync $ */
2/** @file
3 * PGM - Page Manager and Monitor - All context code.
4 */
5
6/*
7 * Copyright (C) 2006-2024 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28
29/*********************************************************************************************************************************
30* Header Files *
31*********************************************************************************************************************************/
32#define LOG_GROUP LOG_GROUP_PGM
33#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
34#ifdef IN_RING0
35# define VBOX_VMM_TARGET_X86
36#endif
37#include <VBox/vmm/pgm.h>
38#include <VBox/vmm/cpum.h>
39#include <VBox/vmm/selm.h>
40#include <VBox/vmm/iem.h>
41#include <VBox/vmm/iom.h>
42#include <VBox/sup.h>
43#include <VBox/vmm/mm.h>
44#include <VBox/vmm/stam.h>
45#include <VBox/vmm/trpm.h>
46#include <VBox/vmm/em.h>
47#include <VBox/vmm/hm.h>
48#include <VBox/vmm/hm_vmx.h>
49#include "PGMInternal.h"
50#include <VBox/vmm/vmcc.h>
51#include "PGMInline.h"
52#include <iprt/assert.h>
53#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
54# include <iprt/asm-amd64-x86.h>
55#endif
56#include <iprt/string.h>
57#include <VBox/log.h>
58#include <VBox/param.h>
59#include <VBox/err.h>
60
61
62/*********************************************************************************************************************************
63* Internal Functions *
64*********************************************************************************************************************************/
65#ifdef VBOX_VMM_TARGET_X86
66DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPT *ppPdpt, PX86PDPAE *ppPD);
67# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
68DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde);
69# endif
70DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3);
71# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
72static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested, PPGMPTWALK pWalk,
73 PPGMPTWALKGST pGstWalk);
74static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3);
75static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT *ppPdpt, PEPTPD *ppPD,
76 PPGMPTWALKGST pGstWalkAll);
77# endif
78static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
79static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT *ppPdpt, PEPTPD *ppPD);
80# ifdef PGM_WITH_PAGE_ZEROING_DETECTION
81static bool pgmHandlePageZeroingCode(PVMCPUCC pVCpu, PCPUMCTX pCtx);
82# endif
83#endif /* VBOX_VMM_TARGET_X86 */
84
85
86#ifdef VBOX_VMM_TARGET_X86
87
88/*
89 * Second level transation - EPT.
90 */
91# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
92# define PGM_SLAT_TYPE PGM_SLAT_TYPE_EPT
93# include "PGMSlatDefs.h"
94# include "PGMAllGstSlatEpt.cpp.h"
95# undef PGM_SLAT_TYPE
96# endif
97
98
99/*
100 * Shadow - 32-bit mode
101 */
102# define PGM_SHW_TYPE PGM_TYPE_32BIT
103# define PGM_SHW_NAME(name) PGM_SHW_NAME_32BIT(name)
104# include "PGMAllShw-x86.cpp.h"
105
106/* Guest - real mode */
107# define PGM_GST_TYPE PGM_TYPE_REAL
108# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
109# define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_REAL(name)
110# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
111# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
112# include "PGMGstDefs-x86.h"
113# include "PGMAllGst-x86.cpp.h"
114# include "PGMAllBth-x86.cpp.h"
115# undef BTH_PGMPOOLKIND_PT_FOR_PT
116# undef BTH_PGMPOOLKIND_ROOT
117# undef PGM_BTH_NAME
118# undef PGM_GST_TYPE
119# undef PGM_GST_NAME
120
121/* Guest - protected mode */
122# define PGM_GST_TYPE PGM_TYPE_PROT
123# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
124# define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name)
125# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
126# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
127# include "PGMGstDefs-x86.h"
128# include "PGMAllGst-x86.cpp.h"
129# include "PGMAllBth-x86.cpp.h"
130# undef BTH_PGMPOOLKIND_PT_FOR_PT
131# undef BTH_PGMPOOLKIND_ROOT
132# undef PGM_BTH_NAME
133# undef PGM_GST_TYPE
134# undef PGM_GST_NAME
135
136/* Guest - 32-bit mode */
137# define PGM_GST_TYPE PGM_TYPE_32BIT
138# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
139# define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_32BIT(name)
140# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
141# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
142# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD
143# include "PGMGstDefs-x86.h"
144# include "PGMAllGst-x86.cpp.h"
145# include "PGMAllBth-x86.cpp.h"
146# undef BTH_PGMPOOLKIND_PT_FOR_BIG
147# undef BTH_PGMPOOLKIND_PT_FOR_PT
148# undef BTH_PGMPOOLKIND_ROOT
149# undef PGM_BTH_NAME
150# undef PGM_GST_TYPE
151# undef PGM_GST_NAME
152
153# undef PGM_SHW_TYPE
154# undef PGM_SHW_NAME
155
156
157/*
158 * Shadow - PAE mode
159 */
160# define PGM_SHW_TYPE PGM_TYPE_PAE
161# define PGM_SHW_NAME(name) PGM_SHW_NAME_PAE(name)
162# define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
163# include "PGMAllShw-x86.cpp.h"
164
165/* Guest - real mode */
166# define PGM_GST_TYPE PGM_TYPE_REAL
167# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
168# define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
169# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
170# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
171# include "PGMGstDefs-x86.h"
172# include "PGMAllBth-x86.cpp.h"
173# undef BTH_PGMPOOLKIND_PT_FOR_PT
174# undef BTH_PGMPOOLKIND_ROOT
175# undef PGM_BTH_NAME
176# undef PGM_GST_TYPE
177# undef PGM_GST_NAME
178
179/* Guest - protected mode */
180# define PGM_GST_TYPE PGM_TYPE_PROT
181# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
182# define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name)
183# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
184# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
185# include "PGMGstDefs-x86.h"
186# include "PGMAllBth-x86.cpp.h"
187# undef BTH_PGMPOOLKIND_PT_FOR_PT
188# undef BTH_PGMPOOLKIND_ROOT
189# undef PGM_BTH_NAME
190# undef PGM_GST_TYPE
191# undef PGM_GST_NAME
192
193/* Guest - 32-bit mode */
194# define PGM_GST_TYPE PGM_TYPE_32BIT
195# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
196# define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_32BIT(name)
197# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
198# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
199# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_FOR_32BIT
200# include "PGMGstDefs-x86.h"
201# include "PGMAllBth-x86.cpp.h"
202# undef BTH_PGMPOOLKIND_PT_FOR_BIG
203# undef BTH_PGMPOOLKIND_PT_FOR_PT
204# undef BTH_PGMPOOLKIND_ROOT
205# undef PGM_BTH_NAME
206# undef PGM_GST_TYPE
207# undef PGM_GST_NAME
208
209
210/* Guest - PAE mode */
211# define PGM_GST_TYPE PGM_TYPE_PAE
212# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
213# define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PAE(name)
214# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
215# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
216# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT
217# include "PGMGstDefs-x86.h"
218# include "PGMAllGst-x86.cpp.h"
219# include "PGMAllBth-x86.cpp.h"
220# undef BTH_PGMPOOLKIND_PT_FOR_BIG
221# undef BTH_PGMPOOLKIND_PT_FOR_PT
222# undef BTH_PGMPOOLKIND_ROOT
223# undef PGM_BTH_NAME
224# undef PGM_GST_TYPE
225# undef PGM_GST_NAME
226
227# undef PGM_SHW_TYPE
228# undef PGM_SHW_NAME
229
230
231/*
232 * Shadow - AMD64 mode
233 */
234# define PGM_SHW_TYPE PGM_TYPE_AMD64
235# define PGM_SHW_NAME(name) PGM_SHW_NAME_AMD64(name)
236# include "PGMAllShw-x86.cpp.h"
237
238/* Guest - protected mode (only used for AMD-V nested paging in 64 bits mode) */
239/** @todo retire this hack. */
240# define PGM_GST_TYPE PGM_TYPE_PROT
241# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
242# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_PROT(name)
243# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
244# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PD_PHYS
245# include "PGMGstDefs-x86.h"
246# include "PGMAllBth-x86.cpp.h"
247# undef BTH_PGMPOOLKIND_PT_FOR_PT
248# undef BTH_PGMPOOLKIND_ROOT
249# undef PGM_BTH_NAME
250# undef PGM_GST_TYPE
251# undef PGM_GST_NAME
252
253# ifdef VBOX_WITH_64_BITS_GUESTS
254/* Guest - AMD64 mode */
255# define PGM_GST_TYPE PGM_TYPE_AMD64
256# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
257# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_AMD64(name)
258# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
259# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
260# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_64BIT_PML4
261# include "PGMGstDefs-x86.h"
262# include "PGMAllGst-x86.cpp.h"
263# include "PGMAllBth-x86.cpp.h"
264# undef BTH_PGMPOOLKIND_PT_FOR_BIG
265# undef BTH_PGMPOOLKIND_PT_FOR_PT
266# undef BTH_PGMPOOLKIND_ROOT
267# undef PGM_BTH_NAME
268# undef PGM_GST_TYPE
269# undef PGM_GST_NAME
270# endif /* VBOX_WITH_64_BITS_GUESTS */
271
272# undef PGM_SHW_TYPE
273# undef PGM_SHW_NAME
274
275
276/*
277 * Shadow - 32-bit nested paging mode.
278 */
279# define PGM_SHW_TYPE PGM_TYPE_NESTED_32BIT
280# define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_32BIT(name)
281# include "PGMAllShw-x86.cpp.h"
282
283/* Guest - real mode */
284# define PGM_GST_TYPE PGM_TYPE_REAL
285# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
286# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_REAL(name)
287# include "PGMGstDefs-x86.h"
288# include "PGMAllBth-x86.cpp.h"
289# undef PGM_BTH_NAME
290# undef PGM_GST_TYPE
291# undef PGM_GST_NAME
292
293/* Guest - protected mode */
294# define PGM_GST_TYPE PGM_TYPE_PROT
295# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
296# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PROT(name)
297# include "PGMGstDefs-x86.h"
298# include "PGMAllBth-x86.cpp.h"
299# undef PGM_BTH_NAME
300# undef PGM_GST_TYPE
301# undef PGM_GST_NAME
302
303/* Guest - 32-bit mode */
304# define PGM_GST_TYPE PGM_TYPE_32BIT
305# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
306# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_32BIT(name)
307# include "PGMGstDefs-x86.h"
308# include "PGMAllBth-x86.cpp.h"
309# undef PGM_BTH_NAME
310# undef PGM_GST_TYPE
311# undef PGM_GST_NAME
312
313/* Guest - PAE mode */
314# define PGM_GST_TYPE PGM_TYPE_PAE
315# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
316# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PAE(name)
317# include "PGMGstDefs-x86.h"
318# include "PGMAllBth-x86.cpp.h"
319# undef PGM_BTH_NAME
320# undef PGM_GST_TYPE
321# undef PGM_GST_NAME
322
323# ifdef VBOX_WITH_64_BITS_GUESTS
324/* Guest - AMD64 mode */
325# define PGM_GST_TYPE PGM_TYPE_AMD64
326# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
327# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_AMD64(name)
328# include "PGMGstDefs-x86.h"
329# include "PGMAllBth-x86.cpp.h"
330# undef PGM_BTH_NAME
331# undef PGM_GST_TYPE
332# undef PGM_GST_NAME
333# endif /* VBOX_WITH_64_BITS_GUESTS */
334
335# undef PGM_SHW_TYPE
336# undef PGM_SHW_NAME
337
338
339/*
340 * Shadow - PAE nested paging mode.
341 */
342# define PGM_SHW_TYPE PGM_TYPE_NESTED_PAE
343# define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_PAE(name)
344# include "PGMAllShw-x86.cpp.h"
345
346/* Guest - real mode */
347# define PGM_GST_TYPE PGM_TYPE_REAL
348# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
349# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_REAL(name)
350# include "PGMGstDefs-x86.h"
351# include "PGMAllBth-x86.cpp.h"
352# undef PGM_BTH_NAME
353# undef PGM_GST_TYPE
354# undef PGM_GST_NAME
355
356/* Guest - protected mode */
357# define PGM_GST_TYPE PGM_TYPE_PROT
358# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
359# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PROT(name)
360# include "PGMGstDefs-x86.h"
361# include "PGMAllBth-x86.cpp.h"
362# undef PGM_BTH_NAME
363# undef PGM_GST_TYPE
364# undef PGM_GST_NAME
365
366/* Guest - 32-bit mode */
367# define PGM_GST_TYPE PGM_TYPE_32BIT
368# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
369# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_32BIT(name)
370# include "PGMGstDefs-x86.h"
371# include "PGMAllBth-x86.cpp.h"
372# undef PGM_BTH_NAME
373# undef PGM_GST_TYPE
374# undef PGM_GST_NAME
375
376/* Guest - PAE mode */
377# define PGM_GST_TYPE PGM_TYPE_PAE
378# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
379# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PAE(name)
380# include "PGMGstDefs-x86.h"
381# include "PGMAllBth-x86.cpp.h"
382# undef PGM_BTH_NAME
383# undef PGM_GST_TYPE
384# undef PGM_GST_NAME
385
386# ifdef VBOX_WITH_64_BITS_GUESTS
387/* Guest - AMD64 mode */
388# define PGM_GST_TYPE PGM_TYPE_AMD64
389# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
390# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_AMD64(name)
391# include "PGMGstDefs-x86.h"
392# include "PGMAllBth-x86.cpp.h"
393# undef PGM_BTH_NAME
394# undef PGM_GST_TYPE
395# undef PGM_GST_NAME
396# endif /* VBOX_WITH_64_BITS_GUESTS */
397
398# undef PGM_SHW_TYPE
399# undef PGM_SHW_NAME
400
401
402/*
403 * Shadow - AMD64 nested paging mode.
404 */
405# define PGM_SHW_TYPE PGM_TYPE_NESTED_AMD64
406# define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_AMD64(name)
407# include "PGMAllShw-x86.cpp.h"
408
409/* Guest - real mode */
410# define PGM_GST_TYPE PGM_TYPE_REAL
411# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
412# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_REAL(name)
413# include "PGMGstDefs-x86.h"
414# include "PGMAllBth-x86.cpp.h"
415# undef PGM_BTH_NAME
416# undef PGM_GST_TYPE
417# undef PGM_GST_NAME
418
419/* Guest - protected mode */
420# define PGM_GST_TYPE PGM_TYPE_PROT
421# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
422# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PROT(name)
423# include "PGMGstDefs-x86.h"
424# include "PGMAllBth-x86.cpp.h"
425# undef PGM_BTH_NAME
426# undef PGM_GST_TYPE
427# undef PGM_GST_NAME
428
429/* Guest - 32-bit mode */
430# define PGM_GST_TYPE PGM_TYPE_32BIT
431# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
432# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_32BIT(name)
433# include "PGMGstDefs-x86.h"
434# include "PGMAllBth-x86.cpp.h"
435# undef PGM_BTH_NAME
436# undef PGM_GST_TYPE
437# undef PGM_GST_NAME
438
439/* Guest - PAE mode */
440# define PGM_GST_TYPE PGM_TYPE_PAE
441# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
442# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PAE(name)
443# include "PGMGstDefs-x86.h"
444# include "PGMAllBth-x86.cpp.h"
445# undef PGM_BTH_NAME
446# undef PGM_GST_TYPE
447# undef PGM_GST_NAME
448
449# ifdef VBOX_WITH_64_BITS_GUESTS
450/* Guest - AMD64 mode */
451# define PGM_GST_TYPE PGM_TYPE_AMD64
452# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
453# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_AMD64(name)
454# include "PGMGstDefs-x86.h"
455# include "PGMAllBth-x86.cpp.h"
456# undef PGM_BTH_NAME
457# undef PGM_GST_TYPE
458# undef PGM_GST_NAME
459# endif /* VBOX_WITH_64_BITS_GUESTS */
460
461# undef PGM_SHW_TYPE
462# undef PGM_SHW_NAME
463
464
465/*
466 * Shadow - EPT.
467 */
468# define PGM_SHW_TYPE PGM_TYPE_EPT
469# define PGM_SHW_NAME(name) PGM_SHW_NAME_EPT(name)
470# include "PGMAllShw-x86.cpp.h"
471
472/* Guest - real mode */
473# define PGM_GST_TYPE PGM_TYPE_REAL
474# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
475# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_REAL(name)
476# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
477# include "PGMGstDefs-x86.h"
478# include "PGMAllBth-x86.cpp.h"
479# undef BTH_PGMPOOLKIND_PT_FOR_PT
480# undef PGM_BTH_NAME
481# undef PGM_GST_TYPE
482# undef PGM_GST_NAME
483
484/* Guest - protected mode */
485# define PGM_GST_TYPE PGM_TYPE_PROT
486# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
487# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name)
488# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
489# include "PGMGstDefs-x86.h"
490# include "PGMAllBth-x86.cpp.h"
491# undef BTH_PGMPOOLKIND_PT_FOR_PT
492# undef PGM_BTH_NAME
493# undef PGM_GST_TYPE
494# undef PGM_GST_NAME
495
496/* Guest - 32-bit mode */
497# define PGM_GST_TYPE PGM_TYPE_32BIT
498# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
499# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_32BIT(name)
500# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
501# include "PGMGstDefs-x86.h"
502# include "PGMAllBth-x86.cpp.h"
503# undef BTH_PGMPOOLKIND_PT_FOR_PT
504# undef PGM_BTH_NAME
505# undef PGM_GST_TYPE
506# undef PGM_GST_NAME
507
508/* Guest - PAE mode */
509# define PGM_GST_TYPE PGM_TYPE_PAE
510# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
511# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PAE(name)
512# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
513# include "PGMGstDefs-x86.h"
514# include "PGMAllBth-x86.cpp.h"
515# undef BTH_PGMPOOLKIND_PT_FOR_PT
516# undef PGM_BTH_NAME
517# undef PGM_GST_TYPE
518# undef PGM_GST_NAME
519
520# ifdef VBOX_WITH_64_BITS_GUESTS
521/* Guest - AMD64 mode */
522# define PGM_GST_TYPE PGM_TYPE_AMD64
523# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
524# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_AMD64(name)
525# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
526# include "PGMGstDefs-x86.h"
527# include "PGMAllBth-x86.cpp.h"
528# undef BTH_PGMPOOLKIND_PT_FOR_PT
529# undef PGM_BTH_NAME
530# undef PGM_GST_TYPE
531# undef PGM_GST_NAME
532# endif /* VBOX_WITH_64_BITS_GUESTS */
533
534# undef PGM_SHW_TYPE
535# undef PGM_SHW_NAME
536
537
538/*
539 * Shadow - NEM / None.
540 */
541# define PGM_SHW_TYPE PGM_TYPE_NONE
542# define PGM_SHW_NAME(name) PGM_SHW_NAME_NONE(name)
543# include "PGMAllShw-x86.cpp.h"
544
545/* Guest - real mode */
546# define PGM_GST_TYPE PGM_TYPE_REAL
547# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
548# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_REAL(name)
549# include "PGMGstDefs-x86.h"
550# include "PGMAllBth-x86.cpp.h"
551# undef PGM_BTH_NAME
552# undef PGM_GST_TYPE
553# undef PGM_GST_NAME
554
555/* Guest - protected mode */
556# define PGM_GST_TYPE PGM_TYPE_PROT
557# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
558# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PROT(name)
559# include "PGMGstDefs-x86.h"
560# include "PGMAllBth-x86.cpp.h"
561# undef PGM_BTH_NAME
562# undef PGM_GST_TYPE
563# undef PGM_GST_NAME
564
565/* Guest - 32-bit mode */
566# define PGM_GST_TYPE PGM_TYPE_32BIT
567# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
568# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_32BIT(name)
569# include "PGMGstDefs-x86.h"
570# include "PGMAllBth-x86.cpp.h"
571# undef PGM_BTH_NAME
572# undef PGM_GST_TYPE
573# undef PGM_GST_NAME
574
575/* Guest - PAE mode */
576# define PGM_GST_TYPE PGM_TYPE_PAE
577# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
578# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PAE(name)
579# include "PGMGstDefs-x86.h"
580# include "PGMAllBth-x86.cpp.h"
581# undef PGM_BTH_NAME
582# undef PGM_GST_TYPE
583# undef PGM_GST_NAME
584
585# ifdef VBOX_WITH_64_BITS_GUESTS
586/* Guest - AMD64 mode */
587# define PGM_GST_TYPE PGM_TYPE_AMD64
588# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
589# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_AMD64(name)
590# include "PGMGstDefs-x86.h"
591# include "PGMAllBth-x86.cpp.h"
592# undef PGM_BTH_NAME
593# undef PGM_GST_TYPE
594# undef PGM_GST_NAME
595# endif /* VBOX_WITH_64_BITS_GUESTS */
596
597# undef PGM_SHW_TYPE
598# undef PGM_SHW_NAME
599
600
601
602/**
603 * Guest mode data array.
604 */
605PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE] =
606{
607 { UINT32_MAX, NULL, NULL, NULL, NULL }, /* 0 */
608 {
609 PGM_TYPE_REAL,
610 PGM_GST_NAME_REAL(GetPage),
611 PGM_GST_NAME_REAL(QueryPageFast),
612 PGM_GST_NAME_REAL(ModifyPage),
613 PGM_GST_NAME_REAL(Enter),
614 PGM_GST_NAME_REAL(Exit),
615# ifdef IN_RING3
616 PGM_GST_NAME_REAL(Relocate),
617# endif
618 },
619 {
620 PGM_TYPE_PROT,
621 PGM_GST_NAME_PROT(GetPage),
622 PGM_GST_NAME_PROT(QueryPageFast),
623 PGM_GST_NAME_PROT(ModifyPage),
624 PGM_GST_NAME_PROT(Enter),
625 PGM_GST_NAME_PROT(Exit),
626# ifdef IN_RING3
627 PGM_GST_NAME_PROT(Relocate),
628# endif
629 },
630 {
631 PGM_TYPE_32BIT,
632 PGM_GST_NAME_32BIT(GetPage),
633 PGM_GST_NAME_32BIT(QueryPageFast),
634 PGM_GST_NAME_32BIT(ModifyPage),
635 PGM_GST_NAME_32BIT(Enter),
636 PGM_GST_NAME_32BIT(Exit),
637# ifdef IN_RING3
638 PGM_GST_NAME_32BIT(Relocate),
639# endif
640 },
641 {
642 PGM_TYPE_PAE,
643 PGM_GST_NAME_PAE(GetPage),
644 PGM_GST_NAME_PAE(QueryPageFast),
645 PGM_GST_NAME_PAE(ModifyPage),
646 PGM_GST_NAME_PAE(Enter),
647 PGM_GST_NAME_PAE(Exit),
648# ifdef IN_RING3
649 PGM_GST_NAME_PAE(Relocate),
650# endif
651 },
652# ifdef VBOX_WITH_64_BITS_GUESTS
653 {
654 PGM_TYPE_AMD64,
655 PGM_GST_NAME_AMD64(GetPage),
656 PGM_GST_NAME_AMD64(QueryPageFast),
657 PGM_GST_NAME_AMD64(ModifyPage),
658 PGM_GST_NAME_AMD64(Enter),
659 PGM_GST_NAME_AMD64(Exit),
660# ifdef IN_RING3
661 PGM_GST_NAME_AMD64(Relocate),
662# endif
663 },
664# endif
665};
666
667
668/**
669 * The shadow mode data array.
670 */
671PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE] =
672{
673 { UINT8_MAX, NULL, NULL, NULL, NULL }, /* 0 */
674 { UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_REAL */
675 { UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_PROT */
676 {
677 PGM_TYPE_32BIT,
678 PGM_SHW_NAME_32BIT(GetPage),
679 PGM_SHW_NAME_32BIT(ModifyPage),
680 PGM_SHW_NAME_32BIT(Enter),
681 PGM_SHW_NAME_32BIT(Exit),
682# ifdef IN_RING3
683 PGM_SHW_NAME_32BIT(Relocate),
684# endif
685 },
686 {
687 PGM_TYPE_PAE,
688 PGM_SHW_NAME_PAE(GetPage),
689 PGM_SHW_NAME_PAE(ModifyPage),
690 PGM_SHW_NAME_PAE(Enter),
691 PGM_SHW_NAME_PAE(Exit),
692# ifdef IN_RING3
693 PGM_SHW_NAME_PAE(Relocate),
694# endif
695 },
696 {
697 PGM_TYPE_AMD64,
698 PGM_SHW_NAME_AMD64(GetPage),
699 PGM_SHW_NAME_AMD64(ModifyPage),
700 PGM_SHW_NAME_AMD64(Enter),
701 PGM_SHW_NAME_AMD64(Exit),
702# ifdef IN_RING3
703 PGM_SHW_NAME_AMD64(Relocate),
704# endif
705 },
706 {
707 PGM_TYPE_NESTED_32BIT,
708 PGM_SHW_NAME_NESTED_32BIT(GetPage),
709 PGM_SHW_NAME_NESTED_32BIT(ModifyPage),
710 PGM_SHW_NAME_NESTED_32BIT(Enter),
711 PGM_SHW_NAME_NESTED_32BIT(Exit),
712# ifdef IN_RING3
713 PGM_SHW_NAME_NESTED_32BIT(Relocate),
714# endif
715 },
716 {
717 PGM_TYPE_NESTED_PAE,
718 PGM_SHW_NAME_NESTED_PAE(GetPage),
719 PGM_SHW_NAME_NESTED_PAE(ModifyPage),
720 PGM_SHW_NAME_NESTED_PAE(Enter),
721 PGM_SHW_NAME_NESTED_PAE(Exit),
722# ifdef IN_RING3
723 PGM_SHW_NAME_NESTED_PAE(Relocate),
724# endif
725 },
726 {
727 PGM_TYPE_NESTED_AMD64,
728 PGM_SHW_NAME_NESTED_AMD64(GetPage),
729 PGM_SHW_NAME_NESTED_AMD64(ModifyPage),
730 PGM_SHW_NAME_NESTED_AMD64(Enter),
731 PGM_SHW_NAME_NESTED_AMD64(Exit),
732# ifdef IN_RING3
733 PGM_SHW_NAME_NESTED_AMD64(Relocate),
734# endif
735 },
736 {
737 PGM_TYPE_EPT,
738 PGM_SHW_NAME_EPT(GetPage),
739 PGM_SHW_NAME_EPT(ModifyPage),
740 PGM_SHW_NAME_EPT(Enter),
741 PGM_SHW_NAME_EPT(Exit),
742# ifdef IN_RING3
743 PGM_SHW_NAME_EPT(Relocate),
744# endif
745 },
746 {
747 PGM_TYPE_NONE,
748 PGM_SHW_NAME_NONE(GetPage),
749 PGM_SHW_NAME_NONE(ModifyPage),
750 PGM_SHW_NAME_NONE(Enter),
751 PGM_SHW_NAME_NONE(Exit),
752# ifdef IN_RING3
753 PGM_SHW_NAME_NONE(Relocate),
754# endif
755 },
756};
757
758
759/**
760 * The guest+shadow mode data array.
761 */
762PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE] =
763{
764# if !defined(IN_RING3) && !defined(VBOX_STRICT)
765# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
766# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
767 { uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler) }
768
769# elif !defined(IN_RING3) && defined(VBOX_STRICT)
770# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
771# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
772 { uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler), Nm(AssertCR3) }
773
774# elif defined(IN_RING3) && !defined(VBOX_STRICT)
775# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL }
776# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
777 { uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), }
778
779# elif defined(IN_RING3) && defined(VBOX_STRICT)
780# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
781# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
782 { uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(AssertCR3) }
783
784# else
785# error "Misconfig."
786# endif
787
788 /* 32-bit shadow paging mode: */
789 PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
790 PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_32BIT_REAL),
791 PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_32BIT_PROT),
792 PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_32BIT_32BIT),
793 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_PAE - illegal */
794 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_AMD64 - illegal */
795 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
796 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_PAE - illegal */
797 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
798 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_EPT - illegal */
799 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NONE - illegal */
800
801 /* PAE shadow paging mode: */
802 PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
803 PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_PAE_REAL),
804 PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_PAE_PROT),
805 PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_PAE_32BIT),
806 PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_PAE_PAE),
807 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_AMD64 - illegal */
808 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_32BIT - illegal */
809 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_PAE - illegal */
810 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
811 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_EPT - illegal */
812 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NONE - illegal */
813
814 /* AMD64 shadow paging mode: */
815 PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
816 PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_AMD64_REAL),
817 PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_AMD64_PROT),
818 PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_AMD64_32BIT),
819 PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_AMD64_PAE),
820# ifdef VBOX_WITH_64_BITS_GUESTS
821 PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_AMD64_AMD64),
822# else
823 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_AMD64 - illegal */
824# endif
825 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
826 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_PAE - illegal */
827 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
828 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_EPT - illegal */
829 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NONE - illegal */
830
831 /* 32-bit nested paging mode: */
832 PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
833 PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_32BIT_REAL),
834 PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_32BIT_PROT),
835 PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_32BIT_32BIT),
836 PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_32BIT_PAE),
837# ifdef VBOX_WITH_64_BITS_GUESTS
838 PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_32BIT_AMD64),
839# else
840 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64 - illegal */
841# endif
842 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
843 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_PAE - illegal */
844 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
845 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_EPT - illegal */
846 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NONE - illegal */
847
848 /* PAE nested paging mode: */
849 PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
850 PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_PAE_REAL),
851 PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_PAE_PROT),
852 PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_PAE_32BIT),
853 PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_PAE_PAE),
854# ifdef VBOX_WITH_64_BITS_GUESTS
855 PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_PAE_AMD64),
856# else
857 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64 - illegal */
858# endif
859 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_32BIT - illegal */
860 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_PAE - illegal */
861 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
862 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_EPT - illegal */
863 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NONE - illegal */
864
865 /* AMD64 nested paging mode: */
866 PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
867 PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_AMD64_REAL),
868 PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_AMD64_PROT),
869 PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_AMD64_32BIT),
870 PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_AMD64_PAE),
871# ifdef VBOX_WITH_64_BITS_GUESTS
872 PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_AMD64_AMD64),
873# else
874 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64 - illegal */
875# endif
876 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
877 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_PAE - illegal */
878 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
879 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_EPT - illegal */
880 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NONE - illegal */
881
882 /* EPT nested paging mode: */
883 PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
884 PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
885 PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
886 PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
887 PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
888# ifdef VBOX_WITH_64_BITS_GUESTS
889 PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
890# else
891 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_AMD64 - illegal */
892# endif
893 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_32BIT - illegal */
894 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_PAE - illegal */
895 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_AMD64 - illegal */
896 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_EPT - illegal */
897 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NONE - illegal */
898
899 /* NONE / NEM: */
900 PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
901 PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
902 PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
903 PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
904 PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
905# ifdef VBOX_WITH_64_BITS_GUESTS
906 PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
907# else
908 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_AMD64 - illegal */
909# endif
910 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_32BIT - illegal */
911 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_PAE - illegal */
912 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_AMD64 - illegal */
913 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_EPT - illegal */
914 PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NONE - illegal */
915
916
917# undef PGMMODEDATABTH_ENTRY
918# undef PGMMODEDATABTH_NULL_ENTRY
919};
920
921
922/** Mask array used by pgmGetCr3MaskForMode.
923 * X86_CR3_AMD64_PAGE_MASK is used for modes that doesn't have a CR3 or EPTP. */
924static uint64_t const g_auCr3MaskForMode[PGMMODE_MAX] =
925{
926 /* [PGMMODE_INVALID] = */ X86_CR3_AMD64_PAGE_MASK,
927 /* [PGMMODE_REAL] = */ X86_CR3_AMD64_PAGE_MASK,
928 /* [PGMMODE_PROTECTED] = */ X86_CR3_AMD64_PAGE_MASK,
929 /* [PGMMODE_32_BIT] = */ X86_CR3_PAGE_MASK,
930 /* [PGMMODE_PAE] = */ X86_CR3_PAE_PAGE_MASK,
931 /* [PGMMODE_PAE_NX] = */ X86_CR3_PAE_PAGE_MASK,
932 /* [PGMMODE_AMD64] = */ X86_CR3_AMD64_PAGE_MASK,
933 /* [PGMMODE_AMD64_NX] = */ X86_CR3_AMD64_PAGE_MASK,
934 /* [PGMMODE_NESTED_32BIT = */ X86_CR3_PAGE_MASK,
935 /* [PGMMODE_NESTED_PAE] = */ X86_CR3_PAE_PAGE_MASK,
936 /* [PGMMODE_NESTED_AMD64] = */ X86_CR3_AMD64_PAGE_MASK,
937 /* [PGMMODE_EPT] = */ X86_CR3_EPT_PAGE_MASK,
938 /* [12] = */ 0,
939 /* [13] = */ 0,
940 /* [14] = */ 0,
941 /* [15] = */ 0,
942 /* [16] = */ 0,
943 /* [17] = */ 0,
944 /* [18] = */ 0,
945 /* [19] = */ 0,
946 /* [20] = */ 0,
947 /* [21] = */ 0,
948 /* [22] = */ 0,
949 /* [23] = */ 0,
950 /* [24] = */ 0,
951 /* [25] = */ 0,
952 /* [26] = */ 0,
953 /* [27] = */ 0,
954 /* [28] = */ 0,
955 /* [29] = */ 0,
956 /* [30] = */ 0,
957 /* [31] = */ 0,
958 /* [PGMMODE_NONE] = */ X86_CR3_AMD64_PAGE_MASK,
959};
960AssertCompile(PGMMODE_NONE == 32);
961
962
963#elif defined(VBOX_VMM_TARGET_ARMV8)
964
965/** @todo Proper ARMv8 guest support for PGM; The idea is to cache the
966 * configuration register values and lazily reconfigure when they
967 * change. */
968# include "PGMAllGst-armv8.cpp.h"
969
970#else
971# error "port me"
972#endif
973
974
975#ifdef VBOX_VMM_TARGET_X86
976
977/**
978 * Gets the physical address mask for CR3 in the given paging mode.
979 *
980 * The mask is for eliminating flags and other stuff in CR3/EPTP when
981 * extracting the physical address. It is not for validating whether there are
982 * reserved bits set. PGM ASSUMES that whoever loaded the CR3 value and passed
983 * it to PGM checked for reserved bits, including reserved physical address
984 * bits.
985 *
986 * @returns The CR3 mask.
987 * @param enmMode The paging mode.
988 * @param enmSlatMode The second-level address translation mode.
989 */
990DECLINLINE(uint64_t) pgmGetCr3MaskForMode(PGMMODE enmMode, PGMSLAT enmSlatMode)
991{
992 if (enmSlatMode == PGMSLAT_DIRECT)
993 {
994 Assert(enmMode != PGMMODE_EPT);
995 return g_auCr3MaskForMode[(unsigned)enmMode < (unsigned)PGMMODE_MAX ? enmMode : 0];
996 }
997 Assert(enmSlatMode == PGMSLAT_EPT);
998 return X86_CR3_EPT_PAGE_MASK;
999}
1000
1001
1002/**
1003 * Gets the masked CR3 value according to the current guest paging mode.
1004 *
1005 * See disclaimer in pgmGetCr3MaskForMode.
1006 *
1007 * @returns The masked PGM CR3 value.
1008 * @param pVCpu The cross context virtual CPU structure.
1009 * @param uCr3 The raw guest CR3 value.
1010 */
1011DECLINLINE(RTGCPHYS) pgmGetGuestMaskedCr3(PVMCPUCC pVCpu, uint64_t uCr3)
1012{
1013 uint64_t const fCr3Mask = pgmGetCr3MaskForMode(pVCpu->pgm.s.enmGuestMode, pVCpu->pgm.s.enmGuestSlatMode);
1014 RTGCPHYS GCPhysCR3 = (RTGCPHYS)(uCr3 & fCr3Mask);
1015 PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
1016 return GCPhysCR3;
1017}
1018
1019
1020# ifdef IN_RING0
1021/**
1022 * \#PF Handler.
1023 *
1024 * @returns VBox status code (appropriate for trap handling and GC return).
1025 * @param pVCpu The cross context virtual CPU structure.
1026 * @param uErr The trap error code.
1027 * @param pCtx Pointer to the register context for the CPU.
1028 * @param pvFault The fault address.
1029 */
1030VMMDECL(int) PGMTrap0eHandler(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault)
1031{
1032 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1033
1034 Log(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv eip=%04x:%RGv cr3=%RGp\n", uErr, pvFault, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, (RTGCPHYS)CPUMGetGuestCR3(pVCpu)));
1035 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.StatRZTrap0e, a);
1036 STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = NULL; } );
1037
1038
1039# ifdef VBOX_WITH_STATISTICS
1040 /*
1041 * Error code stats.
1042 */
1043 if (uErr & X86_TRAP_PF_US)
1044 {
1045 if (!(uErr & X86_TRAP_PF_P))
1046 {
1047 if (uErr & X86_TRAP_PF_RW)
1048 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentWrite);
1049 else
1050 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentRead);
1051 }
1052 else if (uErr & X86_TRAP_PF_RW)
1053 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSWrite);
1054 else if (uErr & X86_TRAP_PF_RSVD)
1055 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSReserved);
1056 else if (uErr & X86_TRAP_PF_ID)
1057 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNXE);
1058 else
1059 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSRead);
1060 }
1061 else
1062 { /* Supervisor */
1063 if (!(uErr & X86_TRAP_PF_P))
1064 {
1065 if (uErr & X86_TRAP_PF_RW)
1066 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentWrite);
1067 else
1068 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentRead);
1069 }
1070 else if (uErr & X86_TRAP_PF_RW)
1071 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVWrite);
1072 else if (uErr & X86_TRAP_PF_ID)
1073 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSNXE);
1074 else if (uErr & X86_TRAP_PF_RSVD)
1075 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVReserved);
1076 }
1077# endif /* VBOX_WITH_STATISTICS */
1078
1079 /*
1080 * Call the worker.
1081 */
1082 uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1083 AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1084 AssertReturn(g_aPgmBothModeData[idxBth].pfnTrap0eHandler, VERR_PGM_MODE_IPE);
1085 bool fLockTaken = false;
1086 int rc = g_aPgmBothModeData[idxBth].pfnTrap0eHandler(pVCpu, uErr, pCtx, pvFault, &fLockTaken);
1087 if (fLockTaken)
1088 {
1089 PGM_LOCK_ASSERT_OWNER(pVM);
1090 PGM_UNLOCK(pVM);
1091 }
1092 LogFlow(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv rc=%Rrc\n", uErr, pvFault, rc));
1093
1094 /*
1095 * Return code tweaks.
1096 */
1097 if (rc != VINF_SUCCESS)
1098 {
1099 if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
1100 rc = VINF_SUCCESS;
1101
1102 /* Note: hack alert for difficult to reproduce problem. */
1103 if ( rc == VERR_PAGE_NOT_PRESENT /* SMP only ; disassembly might fail. */
1104 || rc == VERR_PAGE_TABLE_NOT_PRESENT /* seen with UNI & SMP */
1105 || rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT /* seen with SMP */
1106 || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT) /* precaution */
1107 {
1108 Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGv error code %x (rip=%RGv)\n", rc, pvFault, uErr, pCtx->rip));
1109 /* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about single VCPU VMs though. */
1110 rc = VINF_SUCCESS;
1111 }
1112 }
1113
1114 STAM_STATS({ if (rc == VINF_EM_RAW_GUEST_TRAP) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eGuestPF); });
1115 STAM_STATS({ if (!pVCpu->pgmr0.s.pStatTrap0eAttributionR0)
1116 pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Misc; });
1117 STAM_PROFILE_STOP_EX(&pVCpu->pgm.s.Stats.StatRZTrap0e, pVCpu->pgmr0.s.pStatTrap0eAttributionR0, a);
1118 return rc;
1119}
1120# endif /* IN_RING0 */
1121
1122
1123/**
1124 * Prefetch a page
1125 *
1126 * Typically used to sync commonly used pages before entering raw mode
1127 * after a CR3 reload.
1128 *
1129 * @returns VBox status code suitable for scheduling.
1130 * @retval VINF_SUCCESS on success.
1131 * @retval VINF_PGM_SYNC_CR3 if we're out of shadow pages or something like that.
1132 * @param pVCpu The cross context virtual CPU structure.
1133 * @param GCPtrPage Page to invalidate.
1134 */
1135VMMDECL(int) PGMPrefetchPage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1136{
1137 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
1138
1139 uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1140 AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1141 AssertReturn(g_aPgmBothModeData[idxBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
1142 int rc = g_aPgmBothModeData[idxBth].pfnPrefetchPage(pVCpu, GCPtrPage);
1143
1144 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
1145 AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("rc=%Rrc\n", rc));
1146 return rc;
1147}
1148
1149
1150/**
1151 * Emulation of the invlpg instruction (HC only actually).
1152 *
1153 * @returns Strict VBox status code, special care required.
1154 * @retval VINF_PGM_SYNC_CR3 - handled.
1155 * @retval VINF_EM_RAW_EMULATE_INSTR - not handled (RC only).
1156 *
1157 * @param pVCpu The cross context virtual CPU structure.
1158 * @param GCPtrPage Page to invalidate.
1159 *
1160 * @remark ASSUMES the page table entry or page directory is valid. Fairly
1161 * safe, but there could be edge cases!
1162 *
1163 * @todo Flush page or page directory only if necessary!
1164 * @todo VBOXSTRICTRC
1165 */
1166VMMDECL(int) PGMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1167{
1168 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1169 int rc;
1170 Log3(("PGMInvalidatePage: GCPtrPage=%RGv\n", GCPtrPage));
1171
1172 IEMTlbInvalidatePage(pVCpu, GCPtrPage);
1173
1174 /*
1175 * Call paging mode specific worker.
1176 */
1177 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1178 PGM_LOCK_VOID(pVM);
1179
1180 uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1181 AssertReturnStmt(idxBth < RT_ELEMENTS(g_aPgmBothModeData), PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1182 AssertReturnStmt(g_aPgmBothModeData[idxBth].pfnInvalidatePage, PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1183 rc = g_aPgmBothModeData[idxBth].pfnInvalidatePage(pVCpu, GCPtrPage);
1184
1185 PGM_UNLOCK(pVM);
1186 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1187
1188 /* Ignore all irrelevant error codes. */
1189 if ( rc == VERR_PAGE_NOT_PRESENT
1190 || rc == VERR_PAGE_TABLE_NOT_PRESENT
1191 || rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
1192 || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
1193 rc = VINF_SUCCESS;
1194
1195 return rc;
1196}
1197
1198
1199/**
1200 * Executes an instruction using the interpreter.
1201 *
1202 * @returns VBox status code (appropriate for trap handling and GC return).
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param pvFault Fault address.
1205 */
1206VMMDECL(VBOXSTRICTRC) PGMInterpretInstruction(PVMCPUCC pVCpu, RTGCPTR pvFault)
1207{
1208 RT_NOREF(pvFault);
1209 VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu);
1210 if (rc == VERR_EM_INTERPRETER)
1211 rc = VINF_EM_RAW_EMULATE_INSTR;
1212 if (rc != VINF_SUCCESS)
1213 Log(("PGMInterpretInstruction: returns %Rrc (pvFault=%RGv)\n", VBOXSTRICTRC_VAL(rc), pvFault));
1214 return rc;
1215}
1216
1217
1218/**
1219 * Gets effective page information (from the VMM page directory).
1220 *
1221 * @returns VBox status code.
1222 * @param pVCpu The cross context virtual CPU structure.
1223 * @param GCPtr Guest Context virtual address of the page.
1224 * @param pfFlags Where to store the flags. These are X86_PTE_*.
1225 * @param pHCPhys Where to store the HC physical address of the page.
1226 * This is page aligned.
1227 * @remark You should use PGMMapGetPage() for pages in a mapping.
1228 */
1229VMMDECL(int) PGMShwGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
1230{
1231 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1232 PGM_LOCK_VOID(pVM);
1233
1234 uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1235 AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1236 AssertReturn(g_aPgmShadowModeData[idxShw].pfnGetPage, VERR_PGM_MODE_IPE);
1237 int rc = g_aPgmShadowModeData[idxShw].pfnGetPage(pVCpu, GCPtr, pfFlags, pHCPhys);
1238
1239 PGM_UNLOCK(pVM);
1240 return rc;
1241}
1242
1243
1244/**
1245 * Modify page flags for a range of pages in the shadow context.
1246 *
1247 * The existing flags are ANDed with the fMask and ORed with the fFlags.
1248 *
1249 * @returns VBox status code.
1250 * @param pVCpu The cross context virtual CPU structure.
1251 * @param GCPtr Virtual address of the first page in the range.
1252 * @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
1253 * @param fMask The AND mask - page flags X86_PTE_*.
1254 * Be very CAREFUL when ~'ing constants which could be 32-bit!
1255 * @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1256 * @remark You must use PGMMapModifyPage() for pages in a mapping.
1257 */
1258DECLINLINE(int) pdmShwModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags)
1259{
1260 AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
1261 Assert(!(fOpFlags & ~(PGM_MK_PG_IS_MMIO2 | PGM_MK_PG_IS_WRITE_FAULT)));
1262
1263 GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK; /** @todo this ain't necessary, right... */
1264
1265 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1266 PGM_LOCK_VOID(pVM);
1267
1268 uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1269 AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1270 AssertReturn(g_aPgmShadowModeData[idxShw].pfnModifyPage, VERR_PGM_MODE_IPE);
1271 int rc = g_aPgmShadowModeData[idxShw].pfnModifyPage(pVCpu, GCPtr, GUEST_PAGE_SIZE, fFlags, fMask, fOpFlags);
1272
1273 PGM_UNLOCK(pVM);
1274 return rc;
1275}
1276
1277
1278/**
1279 * Changing the page flags for a single page in the shadow page tables so as to
1280 * make it read-only.
1281 *
1282 * @returns VBox status code.
1283 * @param pVCpu The cross context virtual CPU structure.
1284 * @param GCPtr Virtual address of the first page in the range.
1285 * @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1286 */
1287VMMDECL(int) PGMShwMakePageReadonly(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1288{
1289 return pdmShwModifyPage(pVCpu, GCPtr, 0, ~(uint64_t)X86_PTE_RW, fOpFlags);
1290}
1291
1292
1293/**
1294 * Changing the page flags for a single page in the shadow page tables so as to
1295 * make it writable.
1296 *
1297 * The call must know with 101% certainty that the guest page tables maps this
1298 * as writable too. This function will deal shared, zero and write monitored
1299 * pages.
1300 *
1301 * @returns VBox status code.
1302 * @param pVCpu The cross context virtual CPU structure.
1303 * @param GCPtr Virtual address of the first page in the range.
1304 * @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1305 */
1306VMMDECL(int) PGMShwMakePageWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1307{
1308 if (pVCpu->pgm.s.enmShadowMode != PGMMODE_NONE) /* avoid assertions */
1309 return pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)0, fOpFlags);
1310 return VINF_SUCCESS;
1311}
1312
1313
1314/**
1315 * Changing the page flags for a single page in the shadow page tables so as to
1316 * make it not present.
1317 *
1318 * @returns VBox status code.
1319 * @param pVCpu The cross context virtual CPU structure.
1320 * @param GCPtr Virtual address of the first page in the range.
1321 * @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1322 */
1323VMMDECL(int) PGMShwMakePageNotPresent(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1324{
1325 return pdmShwModifyPage(pVCpu, GCPtr, 0, 0, fOpFlags);
1326}
1327
1328# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
1329
1330/**
1331 * Changing the page flags for a single page in the shadow page tables so as to
1332 * make it supervisor and writable.
1333 *
1334 * This if for dealing with CR0.WP=0 and readonly user pages.
1335 *
1336 * @returns VBox status code.
1337 * @param pVCpu The cross context virtual CPU structure.
1338 * @param GCPtr Virtual address of the first page in the range.
1339 * @param fBigPage Whether or not this is a big page. If it is, we have to
1340 * change the shadow PDE as well. If it isn't, the caller
1341 * has checked that the shadow PDE doesn't need changing.
1342 * We ASSUME 4KB pages backing the big page here!
1343 * @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1344 */
1345int pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags)
1346{
1347 int rc = pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)X86_PTE_US, fOpFlags);
1348 if (rc == VINF_SUCCESS && fBigPage)
1349 {
1350 /* this is a bit ugly... */
1351 switch (pVCpu->pgm.s.enmShadowMode)
1352 {
1353 case PGMMODE_32_BIT:
1354 {
1355 PX86PDE pPde = pgmShwGet32BitPDEPtr(pVCpu, GCPtr);
1356 AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1357 Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1358 pPde->u |= X86_PDE_RW;
1359 Log(("-> PDE=%#llx (32)\n", pPde->u));
1360 break;
1361 }
1362 case PGMMODE_PAE:
1363 case PGMMODE_PAE_NX:
1364 {
1365 PX86PDEPAE pPde = pgmShwGetPaePDEPtr(pVCpu, GCPtr);
1366 AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1367 Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1368 pPde->u |= X86_PDE_RW;
1369 Log(("-> PDE=%#llx (PAE)\n", pPde->u));
1370 break;
1371 }
1372 default:
1373 AssertFailedReturn(VERR_INTERNAL_ERROR_4);
1374 }
1375 }
1376 return rc;
1377}
1378
1379
1380/**
1381 * Gets the shadow page directory for the specified address, PAE.
1382 *
1383 * @returns Pointer to the shadow PD.
1384 * @param pVCpu The cross context virtual CPU structure.
1385 * @param GCPtr The address.
1386 * @param uGstPdpe Guest PDPT entry. Valid.
1387 * @param ppPD Receives address of page directory
1388 */
1389int pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1390{
1391 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1392 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1393 PPGMPOOLPAGE pShwPage;
1394 int rc;
1395 PGM_LOCK_ASSERT_OWNER(pVM);
1396
1397
1398 /* Allocate page directory if not present. */
1399 const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1400 PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1401 PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1402 X86PGPAEUINT const uPdpe = pPdpe->u;
1403 if (uPdpe & (X86_PDPE_P | X86_PDPE_PG_MASK))
1404 {
1405 pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1406 AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1407 Assert((pPdpe->u & X86_PDPE_PG_MASK) == pShwPage->Core.Key);
1408
1409 pgmPoolCacheUsed(pPool, pShwPage);
1410
1411 /* Update the entry if necessary. */
1412 X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key | (uGstPdpe & (X86_PDPE_P | X86_PDPE_A)) | (uPdpe & PGM_PDPT_FLAGS);
1413 if (uPdpeNew == uPdpe)
1414 { /* likely */ }
1415 else
1416 ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1417 }
1418 else
1419 {
1420 RTGCPTR64 GCPdPt;
1421 PGMPOOLKIND enmKind;
1422 if (pVM->pgm.s.fNestedPaging || !CPUMIsGuestPagingEnabled(pVCpu))
1423 {
1424 /* AMD-V nested paging or real/protected mode without paging. */
1425 GCPdPt = GCPtr & ~(RT_BIT_64(X86_PDPT_SHIFT) - 1);
1426 enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1427 }
1428 else if (CPUMGetGuestCR4(pVCpu) & X86_CR4_PAE)
1429 {
1430 if (uGstPdpe & X86_PDPE_P)
1431 {
1432 GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1433 enmKind = PGMPOOLKIND_PAE_PD_FOR_PAE_PD;
1434 }
1435 else
1436 {
1437 /* PD not present; guest must reload CR3 to change it.
1438 * No need to monitor anything in this case. */
1439 /** @todo r=bird: WTF is hit?!? */
1440 /*Assert(VM_IS_RAW_MODE_ENABLED(pVM)); - ??? */
1441 GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1442 enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1443 Assert(uGstPdpe & X86_PDPE_P); /* caller should do this already */
1444 }
1445 }
1446 else
1447 {
1448 GCPdPt = CPUMGetGuestCR3(pVCpu);
1449 enmKind = (PGMPOOLKIND)(PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + iPdPt);
1450 }
1451
1452 /* Create a reference back to the PDPT by using the index in its shadow page. */
1453 rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1454 pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPdPt, false /*fLockPage*/,
1455 &pShwPage);
1456 AssertRCReturn(rc, rc);
1457
1458 /* Hook it up. */
1459 ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key | (uGstPdpe & (X86_PDPE_P | X86_PDPE_A)) | (uPdpe & PGM_PDPT_FLAGS));
1460 }
1461 PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdpe);
1462
1463 *ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1464 return VINF_SUCCESS;
1465}
1466
1467
1468/**
1469 * Gets the pointer to the shadow page directory entry for an address, PAE.
1470 *
1471 * @returns Pointer to the PDE.
1472 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
1473 * @param GCPtr The address.
1474 * @param ppShwPde Receives the address of the pgm pool page for the shadow page directory
1475 */
1476DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde)
1477{
1478 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1479 PGM_LOCK_ASSERT_OWNER(pVM);
1480
1481 PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1482 AssertReturn(pPdpt, VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT); /* can't happen */
1483 const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1484 X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1485 if (!(uPdpe & X86_PDPE_P))
1486 {
1487 LogFlow(("pgmShwGetPaePoolPagePD: PD %d not present (%RX64)\n", iPdPt, uPdpe));
1488 return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1489 }
1490 AssertMsg(uPdpe & X86_PDPE_PG_MASK, ("GCPtr=%RGv\n", GCPtr));
1491
1492 /* Fetch the pgm pool shadow descriptor. */
1493 PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), uPdpe & X86_PDPE_PG_MASK);
1494 AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1495
1496 *ppShwPde = pShwPde;
1497 return VINF_SUCCESS;
1498}
1499
1500
1501/**
1502 * Syncs the SHADOW page directory pointer for the specified address.
1503 *
1504 * Allocates backing pages in case the PDPT or PML4 entry is missing.
1505 *
1506 * The caller is responsible for making sure the guest has a valid PD before
1507 * calling this function.
1508 *
1509 * @returns VBox status code.
1510 * @param pVCpu The cross context virtual CPU structure.
1511 * @param GCPtr The address.
1512 * @param uGstPml4e Guest PML4 entry (valid).
1513 * @param uGstPdpe Guest PDPT entry (valid).
1514 * @param ppPD Receives address of page directory
1515 */
1516static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1517{
1518 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1519 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1520 bool const fNestedPagingOrNoGstPaging = pVM->pgm.s.fNestedPaging || !CPUMIsGuestPagingEnabled(pVCpu);
1521 int rc;
1522
1523 PGM_LOCK_ASSERT_OWNER(pVM);
1524
1525 /*
1526 * PML4.
1527 */
1528 PPGMPOOLPAGE pShwPage;
1529 {
1530 const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1531 PX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1532 AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
1533 X86PGPAEUINT const uPml4e = pPml4e->u;
1534
1535 /* Allocate page directory pointer table if not present. */
1536 if (uPml4e & (X86_PML4E_P | X86_PML4E_PG_MASK))
1537 {
1538 pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1539 AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1540
1541 pgmPoolCacheUsed(pPool, pShwPage);
1542
1543 /* Update the entry if needed. */
1544 X86PGPAEUINT const uPml4eNew = pShwPage->Core.Key | (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1545 | (uPml4e & PGM_PML4_FLAGS);
1546 if (uPml4e == uPml4eNew)
1547 { /* likely */ }
1548 else
1549 ASMAtomicWriteU64(&pPml4e->u, uPml4eNew);
1550 }
1551 else
1552 {
1553 Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1554
1555 RTGCPTR64 GCPml4;
1556 PGMPOOLKIND enmKind;
1557 if (fNestedPagingOrNoGstPaging)
1558 {
1559 /* AMD-V nested paging or real/protected mode without paging */
1560 GCPml4 = (RTGCPTR64)iPml4 << X86_PML4_SHIFT; /** @todo bogus calculation for PML5 */
1561 enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_PHYS;
1562 }
1563 else
1564 {
1565 GCPml4 = uGstPml4e & X86_PML4E_PG_MASK;
1566 enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT;
1567 }
1568
1569 /* Create a reference back to the PDPT by using the index in its shadow page. */
1570 rc = pgmPoolAlloc(pVM, GCPml4, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1571 pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /*fLockPage*/,
1572 &pShwPage);
1573 AssertRCReturn(rc, rc);
1574
1575 /* Hook it up. */
1576 ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1577 | (uPml4e & PGM_PML4_FLAGS));
1578 }
1579 }
1580
1581 /*
1582 * PDPT.
1583 */
1584 const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1585 PX86PDPT pPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1586 PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1587 X86PGPAEUINT const uPdpe = pPdpe->u;
1588
1589 /* Allocate page directory if not present. */
1590 if (uPdpe & (X86_PDPE_P | X86_PDPE_PG_MASK))
1591 {
1592 pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1593 AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1594
1595 pgmPoolCacheUsed(pPool, pShwPage);
1596
1597 /* Update the entry if needed. */
1598 X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key | (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask)
1599 | (uPdpe & PGM_PDPT_FLAGS);
1600 if (uPdpe == uPdpeNew)
1601 { /* likely */ }
1602 else
1603 ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1604 }
1605 else
1606 {
1607 RTGCPTR64 GCPdPt;
1608 PGMPOOLKIND enmKind;
1609 if (fNestedPagingOrNoGstPaging)
1610 {
1611 /* AMD-V nested paging or real/protected mode without paging */
1612 GCPdPt = GCPtr & ~(RT_BIT_64(iPdPt << X86_PDPT_SHIFT) - 1);
1613 enmKind = PGMPOOLKIND_64BIT_PD_FOR_PHYS;
1614 }
1615 else
1616 {
1617 GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1618 enmKind = PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD;
1619 }
1620
1621 /* Create a reference back to the PDPT by using the index in its shadow page. */
1622 rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1623 pShwPage->idx, iPdPt, false /*fLockPage*/,
1624 &pShwPage);
1625 AssertRCReturn(rc, rc);
1626
1627 /* Hook it up. */
1628 ASMAtomicWriteU64(&pPdpe->u,
1629 pShwPage->Core.Key | (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask) | (uPdpe & PGM_PDPT_FLAGS));
1630 }
1631
1632 *ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1633 return VINF_SUCCESS;
1634}
1635
1636
1637/**
1638 * Gets the SHADOW page directory pointer for the specified address (long mode).
1639 *
1640 * @returns VBox status code.
1641 * @param pVCpu The cross context virtual CPU structure.
1642 * @param GCPtr The address.
1643 * @param ppPml4e Receives the address of the page map level 4 entry.
1644 * @param ppPdpt Receives the address of the page directory pointer table.
1645 * @param ppPD Receives the address of the page directory.
1646 */
1647DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPT *ppPdpt, PX86PDPAE *ppPD)
1648{
1649 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1650 PGM_LOCK_ASSERT_OWNER(pVM);
1651
1652 /*
1653 * PML4
1654 */
1655 const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1656 PCX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1657 AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
1658 if (ppPml4e)
1659 *ppPml4e = (PX86PML4E)pPml4e;
1660 X86PGPAEUINT const uPml4e = pPml4e->u;
1661 Log4(("pgmShwGetLongModePDPtr %RGv (%RHv) %RX64\n", GCPtr, pPml4e, uPml4e));
1662 if (!(uPml4e & X86_PML4E_P)) /** @todo other code is check for NULL page frame number! */
1663 return VERR_PAGE_MAP_LEVEL4_NOT_PRESENT;
1664
1665 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1666 PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1667 AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1668
1669 /*
1670 * PDPT
1671 */
1672 const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1673 PCX86PDPT pPdpt = *ppPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1674 X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1675 if (!(uPdpe & X86_PDPE_P)) /** @todo other code is check for NULL page frame number! */
1676 return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1677
1678 pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1679 AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1680
1681 *ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1682 Log4(("pgmShwGetLongModePDPtr %RGv -> *ppPD=%p PDE=%p/%RX64\n", GCPtr, *ppPD, &(*ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK], (*ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK].u));
1683 return VINF_SUCCESS;
1684}
1685
1686
1687/**
1688 * Syncs the SHADOW EPT page directory pointer for the specified address. Allocates
1689 * backing pages in case the PDPT or PML4 entry is missing.
1690 *
1691 * @returns VBox status code.
1692 * @param pVCpu The cross context virtual CPU structure.
1693 * @param GCPtr The address.
1694 * @param ppPdpt Receives address of pdpt
1695 * @param ppPD Receives address of page directory
1696 */
1697static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT *ppPdpt, PEPTPD *ppPD)
1698{
1699 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1700 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1701 int rc;
1702
1703 Assert(pVM->pgm.s.fNestedPaging);
1704 PGM_LOCK_ASSERT_OWNER(pVM);
1705
1706 /*
1707 * PML4 level.
1708 */
1709 PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1710 Assert(pPml4);
1711
1712 /* Allocate page directory pointer table if not present. */
1713 PPGMPOOLPAGE pShwPage;
1714 {
1715 const unsigned iPml4 = (GCPtr >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1716 PEPTPML4E pPml4e = &pPml4->a[iPml4];
1717 EPTPML4E Pml4e;
1718 Pml4e.u = pPml4e->u;
1719 if (!(Pml4e.u & (EPT_E_PG_MASK | EPT_E_READ)))
1720 {
1721 RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT;
1722 rc = pgmPoolAlloc(pVM, GCPml4, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1723 pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /*fLockPage*/,
1724 &pShwPage);
1725 AssertRCReturn(rc, rc);
1726
1727 /* Hook up the new PDPT now. */
1728 ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE);
1729 }
1730 else
1731 {
1732 pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1733 AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1734
1735 pgmPoolCacheUsed(pPool, pShwPage);
1736
1737 /* Hook up the cached PDPT if needed (probably not given 512*512 PTs to sync). */
1738 if (Pml4e.u == (pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE))
1739 { }
1740 else
1741 ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE);
1742 }
1743 }
1744
1745 /*
1746 * PDPT level.
1747 */
1748 const unsigned iPdPt = (GCPtr >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1749 PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1750 PEPTPDPTE pPdpe = &pPdpt->a[iPdPt];
1751
1752 if (ppPdpt)
1753 *ppPdpt = pPdpt;
1754
1755 /* Allocate page directory if not present. */
1756 EPTPDPTE Pdpe;
1757 Pdpe.u = pPdpe->u;
1758 if (!(Pdpe.u & (EPT_E_PG_MASK | EPT_E_READ)))
1759 {
1760 RTGCPTR64 const GCPdPt = GCPtr & ~(RT_BIT_64(EPT_PDPT_SHIFT) - 1);
1761 rc = pgmPoolAlloc(pVM, GCPdPt, PGMPOOLKIND_EPT_PD_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1762 pShwPage->idx, iPdPt, false /*fLockPage*/,
1763 &pShwPage);
1764 AssertRCReturn(rc, rc);
1765
1766 /* Hook up the new PD now. */
1767 ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE);
1768 }
1769 else
1770 {
1771 pShwPage = pgmPoolGetPage(pPool, pPdpe->u & EPT_PDPTE_PG_MASK);
1772 AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1773
1774 pgmPoolCacheUsed(pPool, pShwPage);
1775
1776 /* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
1777 if (Pdpe.u == (pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE))
1778 { }
1779 else
1780 ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE);
1781 }
1782
1783 *ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1784 return VINF_SUCCESS;
1785}
1786
1787
1788# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1789/**
1790 * Syncs the SHADOW nested-guest page directory pointer for the specified address.
1791 * Allocates backing pages in case the PDPT or PML4 entry is missing.
1792 *
1793 * @returns VBox status code.
1794 * @param pVCpu The cross context virtual CPU structure.
1795 * @param GCPhysNested The nested-guest physical address.
1796 * @param ppPdpt Where to store the PDPT. Optional, can be NULL.
1797 * @param ppPD Where to store the PD. Optional, can be NULL.
1798 * @param pGstWalkAll The guest walk info.
1799 */
1800static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT *ppPdpt, PEPTPD *ppPD,
1801 PPGMPTWALKGST pGstWalkAll)
1802{
1803 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1804 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1805 int rc;
1806
1807 PPGMPOOLPAGE pShwPage;
1808 Assert(pVM->pgm.s.fNestedPaging);
1809 Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT);
1810 PGM_LOCK_ASSERT_OWNER(pVM);
1811
1812 /*
1813 * PML4 level.
1814 */
1815 {
1816 PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1817 Assert(pPml4);
1818
1819 /* Allocate page directory pointer table if not present. */
1820 {
1821 uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pml4e.u & pVCpu->pgm.s.fGstEptShadowedPml4eMask;
1822 const unsigned iPml4e = (GCPhysNested >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1823 PEPTPML4E pPml4e = &pPml4->a[iPml4e];
1824
1825 if (!(pPml4e->u & (EPT_E_PG_MASK | EPT_PRESENT_MASK)))
1826 {
1827 RTGCPHYS const GCPhysPdpt = pGstWalkAll->u.Ept.Pml4e.u & EPT_PML4E_PG_MASK;
1828 rc = pgmPoolAlloc(pVM, GCPhysPdpt, PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT, PGMPOOLACCESS_DONTCARE,
1829 PGM_A20_IS_ENABLED(pVCpu), pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4e, false /*fLockPage*/,
1830 &pShwPage);
1831 AssertRCReturn(rc, rc);
1832
1833 /* Hook up the new PDPT now. */
1834 ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | fShwFlags);
1835 }
1836 else
1837 {
1838 pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1839 AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1840
1841 pgmPoolCacheUsed(pPool, pShwPage);
1842
1843 /* Hook up the cached PDPT if needed (probably not given 512*512 PTs to sync). */
1844 if (pPml4e->u != (pShwPage->Core.Key | fShwFlags))
1845 ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | fShwFlags);
1846 }
1847 Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
1848 Log7Func(("GstPml4e=%RX64 ShwPml4e=%RX64 iPml4e=%u\n", pGstWalkAll->u.Ept.Pml4e.u, pPml4e->u, iPml4e));
1849 }
1850 }
1851
1852 /*
1853 * PDPT level.
1854 */
1855 {
1856 AssertReturn(!(pGstWalkAll->u.Ept.Pdpte.u & EPT_E_LEAF), VERR_NOT_SUPPORTED); /* shadowing 1GB pages not supported yet. */
1857
1858 PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1859 if (ppPdpt)
1860 *ppPdpt = pPdpt;
1861
1862 uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pdpte.u & pVCpu->pgm.s.fGstEptShadowedPdpteMask;
1863 const unsigned iPdPte = (GCPhysNested >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1864 PEPTPDPTE pPdpte = &pPdpt->a[iPdPte];
1865
1866 if (!(pPdpte->u & (EPT_E_PG_MASK | EPT_PRESENT_MASK)))
1867 {
1868 RTGCPHYS const GCPhysPd = pGstWalkAll->u.Ept.Pdpte.u & EPT_PDPTE_PG_MASK;
1869 rc = pgmPoolAlloc(pVM, GCPhysPd, PGMPOOLKIND_EPT_PD_FOR_EPT_PD, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1870 pShwPage->idx, iPdPte, false /*fLockPage*/, &pShwPage);
1871 AssertRCReturn(rc, rc);
1872
1873 /* Hook up the new PD now. */
1874 ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key | fShwFlags);
1875 }
1876 else
1877 {
1878 pShwPage = pgmPoolGetPage(pPool, pPdpte->u & EPT_PDPTE_PG_MASK);
1879 AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1880
1881 pgmPoolCacheUsed(pPool, pShwPage);
1882
1883 /* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
1884 if (pPdpte->u != (pShwPage->Core.Key | fShwFlags))
1885 ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key | fShwFlags);
1886 }
1887 Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
1888 Log7Func(("GstPdpte=%RX64 ShwPdpte=%RX64 iPdPte=%u \n", pGstWalkAll->u.Ept.Pdpte.u, pPdpte->u, iPdPte));
1889
1890 *ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1891 }
1892
1893 return VINF_SUCCESS;
1894}
1895# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
1896
1897
1898# ifdef IN_RING0
1899/**
1900 * Synchronizes a range of nested page table entries.
1901 *
1902 * The caller must own the PGM lock.
1903 *
1904 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
1905 * @param GCPhys Where to start.
1906 * @param cPages How many pages which entries should be synced.
1907 * @param enmShwPagingMode The shadow paging mode (PGMMODE_EPT for VT-x,
1908 * host paging mode for AMD-V).
1909 */
1910int pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint32_t cPages, PGMMODE enmShwPagingMode)
1911{
1912 PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
1913
1914/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
1915 int rc;
1916 switch (enmShwPagingMode)
1917 {
1918 case PGMMODE_32_BIT:
1919 {
1920 X86PDE PdeDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
1921 rc = PGM_BTH_NAME_32BIT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /*uErr*/);
1922 break;
1923 }
1924
1925 case PGMMODE_PAE:
1926 case PGMMODE_PAE_NX:
1927 {
1928 X86PDEPAE PdeDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
1929 rc = PGM_BTH_NAME_PAE_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /*uErr*/);
1930 break;
1931 }
1932
1933 case PGMMODE_AMD64:
1934 case PGMMODE_AMD64_NX:
1935 {
1936 X86PDEPAE PdeDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
1937 rc = PGM_BTH_NAME_AMD64_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /*uErr*/);
1938 break;
1939 }
1940
1941 case PGMMODE_EPT:
1942 {
1943 X86PDEPAE PdeDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
1944 rc = PGM_BTH_NAME_EPT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /*uErr*/);
1945 break;
1946 }
1947
1948 default:
1949 AssertMsgFailedReturn(("%d\n", enmShwPagingMode), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
1950 }
1951 return rc;
1952}
1953# endif /* IN_RING0 */
1954
1955# endif /* !VBOX_WITH_ONLY_PGM_NEM_MODE */
1956
1957/**
1958 * Maps the guest CR3.
1959 *
1960 * @returns VBox status code.
1961 * @param pVCpu The cross context virtual CPU structure.
1962 * @param GCPhysCr3 The guest CR3 value.
1963 * @param pHCPtrGuestCr3 Where to store the mapped memory.
1964 */
1965DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3)
1966{
1967 /** @todo this needs some reworking wrt. locking? */
1968 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1969 PGM_LOCK_VOID(pVM);
1970 PPGMPAGE pPageCr3 = pgmPhysGetPage(pVM, GCPhysCr3);
1971 AssertReturnStmt(pPageCr3, PGM_UNLOCK(pVM), VERR_PGM_INVALID_CR3_ADDR);
1972
1973 RTHCPTR HCPtrGuestCr3;
1974 int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCr3, GCPhysCr3, (void **)&HCPtrGuestCr3);
1975 PGM_UNLOCK(pVM);
1976
1977 *pHCPtrGuestCr3 = HCPtrGuestCr3;
1978 return rc;
1979}
1980
1981
1982# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1983/**
1984 * Unmaps the guest CR3.
1985 *
1986 * @returns VBox status code.
1987 * @param pVCpu The cross context virtual CPU structure.
1988 */
1989DECLINLINE(int) pgmGstUnmapCr3(PVMCPUCC pVCpu)
1990{
1991 uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1992 AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1993 AssertReturn(g_aPgmBothModeData[idxBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
1994 return g_aPgmBothModeData[idxBth].pfnUnmapCR3(pVCpu);
1995}
1996# endif
1997
1998#endif /* VBOX_VMM_TARGET_X86 */
1999
2000
2001/**
2002 * Gets effective Guest OS page information.
2003 *
2004 * @returns VBox status code.
2005 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
2006 * @param GCPtr Guest Context virtual address of the page.
2007 * @param pWalk Where to store the page walk information.
2008 * @thread EMT(pVCpu)
2009 */
2010VMMDECL(int) PGMGstGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk)
2011{
2012 VMCPU_ASSERT_EMT(pVCpu);
2013 Assert(pWalk);
2014#ifdef VBOX_VMM_TARGET_X86
2015 uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
2016 AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
2017 AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
2018 return g_aPgmGuestModeData[idx].pfnGetPage(pVCpu, GCPtr, pWalk);
2019
2020#elif defined(VBOX_VMM_TARGET_ARMV8)
2021 return pgmGstGetPageArmv8Hack(pVCpu, GCPtr, pWalk);
2022
2023#else
2024# error "port me"
2025#endif
2026}
2027
2028
2029#ifdef VBOX_VMM_TARGET_X86 /** @todo Implement PGMGstQueryPageFast for ARMv8! */
2030/**
2031 * Gets effective Guest OS page information.
2032 *
2033 * @returns VBox status code.
2034 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
2035 * @param GCPtr Guest Context virtual address of the page.
2036 * @param fFlags PGMQPAGE_F_XXX. If zero, no accessed or dirty bits will
2037 * be set.
2038 * @param pWalk Where to store the page walk information.
2039 * @thread EMT(pVCpu)
2040 */
2041VMM_INT_DECL(int) PGMGstQueryPageFast(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fFlags, PPGMPTWALKFAST pWalk)
2042{
2043 VMCPU_ASSERT_EMT(pVCpu);
2044 Assert(pWalk);
2045 Assert(!(fFlags & ~(PGMQPAGE_F_VALID_MASK)));
2046 Assert(!(fFlags & PGMQPAGE_F_EXECUTE) || !(fFlags & PGMQPAGE_F_WRITE));
2047 uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
2048 AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
2049 AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
2050 return g_aPgmGuestModeData[idx].pfnQueryPageFast(pVCpu, GCPtr, fFlags, pWalk);
2051}
2052#endif /* VBOX_VMM_TARGET_X86 */
2053
2054
2055/**
2056 * Performs a guest page table walk.
2057 *
2058 * The guest should be in paged protect mode or long mode when making a call to
2059 * this function.
2060 *
2061 * @returns VBox status code.
2062 * @retval VINF_SUCCESS on success.
2063 * @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
2064 * @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
2065 * not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
2066 *
2067 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
2068 * @param GCPtr The guest virtual address to walk by.
2069 * @param pWalk Where to return the walk result. This is valid for some
2070 * error codes as well.
2071 * @param pGstWalk The guest mode specific page walk information.
2072 */
2073int pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
2074{
2075 VMCPU_ASSERT_EMT(pVCpu);
2076#ifdef VBOX_VMM_TARGET_X86
2077 switch (pVCpu->pgm.s.enmGuestMode)
2078 {
2079 case PGMMODE_32_BIT:
2080 pGstWalk->enmType = PGMPTWALKGSTTYPE_32BIT;
2081 return PGM_GST_NAME_32BIT(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Legacy);
2082
2083 case PGMMODE_PAE:
2084 case PGMMODE_PAE_NX:
2085 pGstWalk->enmType = PGMPTWALKGSTTYPE_PAE;
2086 return PGM_GST_NAME_PAE(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Pae);
2087
2088 case PGMMODE_AMD64:
2089 case PGMMODE_AMD64_NX:
2090 pGstWalk->enmType = PGMPTWALKGSTTYPE_AMD64;
2091 return PGM_GST_NAME_AMD64(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Amd64);
2092
2093 case PGMMODE_REAL:
2094 case PGMMODE_PROTECTED:
2095 pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2096 return VERR_PGM_NOT_USED_IN_MODE;
2097
2098 case PGMMODE_EPT:
2099 case PGMMODE_NESTED_32BIT:
2100 case PGMMODE_NESTED_PAE:
2101 case PGMMODE_NESTED_AMD64:
2102 default:
2103 AssertFailed();
2104 pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2105 return VERR_PGM_NOT_USED_IN_MODE;
2106 }
2107
2108#elif defined(VBOX_VMM_TARGET_ARMV8)
2109 /** @todo temporary hack. */
2110 RT_NOREF(pGstWalk);
2111 return pgmGstGetPageArmv8Hack(pVCpu, GCPtr, pWalk);
2112
2113#else
2114# error "port me"
2115#endif
2116}
2117
2118
2119/**
2120 * Tries to continue the previous walk.
2121 *
2122 * @note Requires the caller to hold the PGM lock from the first
2123 * pgmGstPtWalk() call to the last pgmGstPtWalkNext() call. Otherwise
2124 * we cannot use the pointers.
2125 *
2126 * @returns VBox status code.
2127 * @retval VINF_SUCCESS on success.
2128 * @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
2129 * @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
2130 * not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
2131 *
2132 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
2133 * @param GCPtr The guest virtual address to walk by.
2134 * @param pWalk Pointer to the previous walk result and where to return
2135 * the result of this walk. This is valid for some error
2136 * codes as well.
2137 * @param pGstWalk The guest-mode specific walk information.
2138 */
2139int pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
2140{
2141#ifdef VBOX_VMM_TARGET_X86 /** @todo optimize for ARMv8 */
2142 /*
2143 * We can only handle successfully walks.
2144 * We also limit ourselves to the next page.
2145 */
2146 if ( pWalk->fSucceeded
2147 && GCPtr - pWalk->GCPtr == GUEST_PAGE_SIZE)
2148 {
2149 Assert(pWalk->uLevel == 0);
2150 if (pGstWalk->enmType == PGMPTWALKGSTTYPE_AMD64)
2151 {
2152 /*
2153 * AMD64
2154 */
2155 if (!pWalk->fGigantPage && !pWalk->fBigPage)
2156 {
2157 /*
2158 * We fall back to full walk if the PDE table changes, if any
2159 * reserved bits are set, or if the effective page access changes.
2160 */
2161 const uint64_t fPteSame = X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_PWT
2162 | X86_PTE_PCD | X86_PTE_A | X86_PTE_PAE_NX;
2163 const uint64_t fPdeSame = X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT
2164 | X86_PDE_PCD | X86_PDE_A | X86_PDE_PAE_NX | X86_PDE_PS;
2165
2166 if ((GCPtr >> X86_PD_PAE_SHIFT) == (pWalk->GCPtr >> X86_PD_PAE_SHIFT))
2167 {
2168 if (pGstWalk->u.Amd64.pPte)
2169 {
2170 X86PTEPAE Pte;
2171 Pte.u = pGstWalk->u.Amd64.pPte[1].u;
2172 if ( (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame)
2173 && !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2174 {
2175 pWalk->GCPtr = GCPtr;
2176 pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2177 pGstWalk->u.Amd64.Pte.u = Pte.u;
2178 pGstWalk->u.Amd64.pPte++;
2179 return VINF_SUCCESS;
2180 }
2181 }
2182 }
2183 else if ((GCPtr >> X86_PDPT_SHIFT) == (pWalk->GCPtr >> X86_PDPT_SHIFT))
2184 {
2185 Assert(!((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK)); /* Must be first PT entry. */
2186 if (pGstWalk->u.Amd64.pPde)
2187 {
2188 X86PDEPAE Pde;
2189 Pde.u = pGstWalk->u.Amd64.pPde[1].u;
2190 if ( (Pde.u & fPdeSame) == (pGstWalk->u.Amd64.Pde.u & fPdeSame)
2191 && !(Pde.u & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask))
2192 {
2193 /* Get the new PTE and check out the first entry. */
2194 int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, PGM_A20_APPLY(pVCpu, (Pde.u & X86_PDE_PAE_PG_MASK)),
2195 &pGstWalk->u.Amd64.pPt);
2196 if (RT_SUCCESS(rc))
2197 {
2198 pGstWalk->u.Amd64.pPte = &pGstWalk->u.Amd64.pPt->a[0];
2199 X86PTEPAE Pte;
2200 Pte.u = pGstWalk->u.Amd64.pPte->u;
2201 if ( (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame)
2202 && !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2203 {
2204 pWalk->GCPtr = GCPtr;
2205 pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2206 pGstWalk->u.Amd64.Pte.u = Pte.u;
2207 pGstWalk->u.Amd64.Pde.u = Pde.u;
2208 pGstWalk->u.Amd64.pPde++;
2209 return VINF_SUCCESS;
2210 }
2211 }
2212 }
2213 }
2214 }
2215 }
2216 else if (!pWalk->fGigantPage)
2217 {
2218 if ((GCPtr & X86_PAGE_2M_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_2M_BASE_MASK))
2219 {
2220 pWalk->GCPtr = GCPtr;
2221 pWalk->GCPhys += GUEST_PAGE_SIZE;
2222 return VINF_SUCCESS;
2223 }
2224 }
2225 else
2226 {
2227 if ((GCPtr & X86_PAGE_1G_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_1G_BASE_MASK))
2228 {
2229 pWalk->GCPtr = GCPtr;
2230 pWalk->GCPhys += GUEST_PAGE_SIZE;
2231 return VINF_SUCCESS;
2232 }
2233 }
2234 }
2235 }
2236#endif /* VBOX_VMM_TARGET_X86 */
2237 /* Case we don't handle. Do full walk. */
2238 return pgmGstPtWalk(pVCpu, GCPtr, pWalk, pGstWalk);
2239}
2240
2241
2242#ifdef VBOX_VMM_TARGET_X86
2243/**
2244 * Modify page flags for a range of pages in the guest's tables
2245 *
2246 * The existing flags are ANDed with the fMask and ORed with the fFlags.
2247 *
2248 * @returns VBox status code.
2249 * @param pVCpu The cross context virtual CPU structure.
2250 * @param GCPtr Virtual address of the first page in the range.
2251 * @param cb Size (in bytes) of the range to apply the modification to.
2252 * @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
2253 * @param fMask The AND mask - page flags X86_PTE_*, excluding the page mask of course.
2254 * Be very CAREFUL when ~'ing constants which could be 32-bit!
2255 */
2256VMMDECL(int) PGMGstModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
2257{
2258 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2259 VMCPU_ASSERT_EMT(pVCpu);
2260
2261 /*
2262 * Validate input.
2263 */
2264 AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
2265 Assert(cb);
2266
2267 LogFlow(("PGMGstModifyPage %RGv %d bytes fFlags=%08llx fMask=%08llx\n", GCPtr, cb, fFlags, fMask));
2268
2269 /*
2270 * Adjust input.
2271 */
2272 cb += GCPtr & GUEST_PAGE_OFFSET_MASK;
2273 cb = RT_ALIGN_Z(cb, GUEST_PAGE_SIZE);
2274 GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK;
2275
2276 /*
2277 * Call worker.
2278 */
2279 uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
2280 AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
2281 AssertReturn(g_aPgmGuestModeData[idx].pfnModifyPage, VERR_PGM_MODE_IPE);
2282 int rc = g_aPgmGuestModeData[idx].pfnModifyPage(pVCpu, GCPtr, cb, fFlags, fMask);
2283
2284 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2285 return rc;
2286}
2287#endif /* VBOX_VMM_TARGET_X86 */
2288
2289#ifdef VBOX_VMM_TARGET_X86
2290
2291/**
2292 * Checks whether the given PAE PDPEs are potentially valid for the guest.
2293 *
2294 * @returns @c true if the PDPE is valid, @c false otherwise.
2295 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
2296 * @param paPaePdpes The PAE PDPEs to validate.
2297 *
2298 * @remarks This function -only- checks the reserved bits in the PDPE entries.
2299 */
2300VMM_INT_DECL(bool) PGMGstArePaePdpesValid(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
2301{
2302 Assert(paPaePdpes);
2303 for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
2304 {
2305 X86PDPE const PaePdpe = paPaePdpes[i];
2306 if ( !(PaePdpe.u & X86_PDPE_P)
2307 || !(PaePdpe.u & pVCpu->pgm.s.fGstPaeMbzPdpeMask))
2308 { /* likely */ }
2309 else
2310 return false;
2311 }
2312 return true;
2313}
2314
2315
2316/**
2317 * Performs the lazy mapping of the 32-bit guest PD.
2318 *
2319 * @returns VBox status code.
2320 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
2321 * @param ppPd Where to return the pointer to the mapping. This is
2322 * always set.
2323 */
2324int pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd)
2325{
2326 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2327 PGM_LOCK_VOID(pVM);
2328
2329 Assert(!pVCpu->pgm.s.CTX_SUFF(pGst32BitPd));
2330
2331 RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2332 PPGMPAGE pPage;
2333 int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2334 if (RT_SUCCESS(rc))
2335 {
2336 rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPd);
2337 if (RT_SUCCESS(rc))
2338 {
2339# ifdef IN_RING3
2340 pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
2341 pVCpu->pgm.s.pGst32BitPdR3 = *ppPd;
2342# else
2343 pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR0PTR;
2344 pVCpu->pgm.s.pGst32BitPdR0 = *ppPd;
2345# endif
2346 PGM_UNLOCK(pVM);
2347 return VINF_SUCCESS;
2348 }
2349 AssertRC(rc);
2350 }
2351 PGM_UNLOCK(pVM);
2352
2353 *ppPd = NULL;
2354 return rc;
2355}
2356
2357
2358/**
2359 * Performs the lazy mapping of the PAE guest PDPT.
2360 *
2361 * @returns VBox status code.
2362 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
2363 * @param ppPdpt Where to return the pointer to the mapping. This is
2364 * always set.
2365 */
2366int pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt)
2367{
2368 Assert(!pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt));
2369 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2370 PGM_LOCK_VOID(pVM);
2371
2372 RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2373 PPGMPAGE pPage;
2374 int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2375 if (RT_SUCCESS(rc))
2376 {
2377 rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPdpt);
2378 if (RT_SUCCESS(rc))
2379 {
2380# ifdef IN_RING3
2381 pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
2382 pVCpu->pgm.s.pGstPaePdptR3 = *ppPdpt;
2383# else
2384 pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
2385 pVCpu->pgm.s.pGstPaePdptR0 = *ppPdpt;
2386# endif
2387 PGM_UNLOCK(pVM);
2388 return VINF_SUCCESS;
2389 }
2390 AssertRC(rc);
2391 }
2392
2393 PGM_UNLOCK(pVM);
2394 *ppPdpt = NULL;
2395 return rc;
2396}
2397
2398
2399/**
2400 * Performs the lazy mapping / updating of a PAE guest PD.
2401 *
2402 * @returns Pointer to the mapping.
2403 * @returns VBox status code.
2404 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
2405 * @param iPdpt Which PD entry to map (0..3).
2406 * @param ppPd Where to return the pointer to the mapping. This is
2407 * always set.
2408 */
2409int pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd)
2410{
2411 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2412 PGM_LOCK_VOID(pVM);
2413
2414 PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
2415 Assert(pGuestPDPT);
2416 Assert(pGuestPDPT->a[iPdpt].u & X86_PDPE_P);
2417 RTGCPHYS GCPhys = pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK;
2418 bool const fChanged = pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] != GCPhys;
2419
2420 PPGMPAGE pPage;
2421 int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2422 if (RT_SUCCESS(rc))
2423 {
2424 rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)ppPd);
2425 AssertRC(rc);
2426 if (RT_SUCCESS(rc))
2427 {
2428# ifdef IN_RING3
2429 pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2430 pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = *ppPd;
2431# else
2432 pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2433 pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = *ppPd;
2434# endif
2435 if (fChanged)
2436 pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = GCPhys;
2437 PGM_UNLOCK(pVM);
2438 return VINF_SUCCESS;
2439 }
2440 }
2441
2442 /* Invalid page or some failure, invalidate the entry. */
2443 pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = NIL_RTGCPHYS;
2444 pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2445 pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2446
2447 PGM_UNLOCK(pVM);
2448 return rc;
2449}
2450
2451
2452/**
2453 * Performs the lazy mapping of the 32-bit guest PD.
2454 *
2455 * @returns VBox status code.
2456 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
2457 * @param ppPml4 Where to return the pointer to the mapping. This will
2458 * always be set.
2459 */
2460int pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4)
2461{
2462 Assert(!pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4));
2463 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2464 PGM_LOCK_VOID(pVM);
2465
2466 RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2467 PPGMPAGE pPage;
2468 int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2469 if (RT_SUCCESS(rc))
2470 {
2471 rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPml4);
2472 if (RT_SUCCESS(rc))
2473 {
2474# ifdef IN_RING3
2475 pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
2476 pVCpu->pgm.s.pGstAmd64Pml4R3 = *ppPml4;
2477# else
2478 pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
2479 pVCpu->pgm.s.pGstAmd64Pml4R0 = *ppPml4;
2480# endif
2481 PGM_UNLOCK(pVM);
2482 return VINF_SUCCESS;
2483 }
2484 }
2485
2486 PGM_UNLOCK(pVM);
2487 *ppPml4 = NULL;
2488 return rc;
2489}
2490
2491
2492# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2493 /**
2494 * Performs the lazy mapping of the guest PML4 table when using EPT paging.
2495 *
2496 * @returns VBox status code.
2497 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
2498 * @param ppEptPml4 Where to return the pointer to the mapping. This will
2499 * always be set.
2500 */
2501int pgmGstLazyMapEptPml4(PVMCPUCC pVCpu, PEPTPML4 *ppEptPml4)
2502{
2503 Assert(!pVCpu->pgm.s.CTX_SUFF(pGstEptPml4));
2504 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2505 PGM_LOCK_VOID(pVM);
2506
2507 RTGCPHYS const GCPhysEpt = pVCpu->pgm.s.uEptPtr & EPT_EPTP_PG_MASK;
2508 PPGMPAGE pPage;
2509 int rc = pgmPhysGetPageEx(pVM, GCPhysEpt, &pPage);
2510 if (RT_SUCCESS(rc))
2511 {
2512 rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysEpt, (void **)ppEptPml4);
2513 if (RT_SUCCESS(rc))
2514 {
2515# ifdef IN_RING3
2516 pVCpu->pgm.s.pGstEptPml4R0 = NIL_RTR0PTR;
2517 pVCpu->pgm.s.pGstEptPml4R3 = *ppEptPml4;
2518# else
2519 pVCpu->pgm.s.pGstEptPml4R3 = NIL_RTR3PTR;
2520 pVCpu->pgm.s.pGstEptPml4R0 = *ppEptPml4;
2521# endif
2522 PGM_UNLOCK(pVM);
2523 return VINF_SUCCESS;
2524 }
2525 }
2526
2527 PGM_UNLOCK(pVM);
2528 *ppEptPml4 = NULL;
2529 return rc;
2530}
2531# endif
2532
2533
2534/**
2535 * Gets the current CR3 register value for the shadow memory context.
2536 * @returns CR3 value.
2537 * @param pVCpu The cross context virtual CPU structure.
2538 */
2539VMMDECL(RTHCPHYS) PGMGetHyperCR3(PVMCPU pVCpu)
2540{
2541# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2542 PPGMPOOLPAGE pPoolPage = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2543 AssertPtrReturn(pPoolPage, NIL_RTHCPHYS);
2544 return pPoolPage->Core.Key;
2545# else
2546 RT_NOREF(pVCpu);
2547 return NIL_RTHCPHYS;
2548# endif
2549}
2550
2551
2552/**
2553 * Forces lazy remapping of the guest's PAE page-directory structures.
2554 *
2555 * @param pVCpu The cross context virtual CPU structure.
2556 */
2557static void pgmGstFlushPaePdpes(PVMCPU pVCpu)
2558{
2559 for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.aGCPhysGstPaePDs); i++)
2560 {
2561 pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2562 pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2563 pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2564 }
2565}
2566
2567
2568# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2569
2570/**
2571 * Performs a guest second-level address translation (SLAT).
2572 *
2573 * @returns VBox status code.
2574 * @retval VINF_SUCCESS on success.
2575 * @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
2576 * @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
2577 * not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
2578 *
2579 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
2580 * @param GCPhysNested The nested-guest physical address being translated.
2581 * @param fIsLinearAddrValid Whether the linear address in @a GCPtrNested is the
2582 * cause for this translation.
2583 * @param GCPtrNested The nested-guest virtual address that initiated the
2584 * SLAT. If none, pass 0 (and not NIL_RTGCPTR).
2585 * @param pWalk Where to return the walk result. This is updated for
2586 * all error codes other than
2587 * VERR_PGM_NOT_USED_IN_MODE.
2588 * @param pGstWalk Where to store the second-level paging-mode specific
2589 * walk info.
2590 */
2591static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested,
2592 PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
2593{
2594 /* SLAT mode must be valid at this point as this should only be used -after- we have determined SLAT mode. */
2595 Assert( pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_DIRECT
2596 && pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_INVALID);
2597 AssertPtr(pWalk);
2598 AssertPtr(pGstWalk);
2599 switch (pVCpu->pgm.s.enmGuestSlatMode)
2600 {
2601 case PGMSLAT_EPT:
2602 pGstWalk->enmType = PGMPTWALKGSTTYPE_EPT;
2603 return PGM_GST_SLAT_NAME_EPT(Walk)(pVCpu, GCPhysNested, fIsLinearAddrValid, GCPtrNested, pWalk, &pGstWalk->u.Ept);
2604
2605 default:
2606 AssertFailed();
2607 pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2608 return VERR_PGM_NOT_USED_IN_MODE;
2609 }
2610}
2611
2612
2613/**
2614 * Performs second-level address translation for the given CR3 and updates the
2615 * nested-guest CR3 when successful.
2616 *
2617 * @returns VBox status code.
2618 * @param pVCpu The cross context virtual CPU structure.
2619 * @param uCr3 The masked nested-guest CR3 value.
2620 * @param pGCPhysCR3 Where to store the translated CR3.
2621 *
2622 * @warning This updates PGMCPU::GCPhysNstGstCR3 when the translation succeeds. Be
2623 * mindful of this in code that's hyper sensitive to the order of
2624 * operations.
2625 */
2626static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3)
2627{
2628 if (uCr3 != pVCpu->pgm.s.GCPhysNstGstCR3)
2629 {
2630 PGMPTWALK Walk;
2631 PGMPTWALKGST GstWalk;
2632 int const rc = pgmGstSlatWalk(pVCpu, uCr3, false /* fIsLinearAddrValid */, 0 /* GCPtrNested */, &Walk, &GstWalk);
2633 if (RT_SUCCESS(rc))
2634 {
2635 /* Update nested-guest CR3. */
2636 pVCpu->pgm.s.GCPhysNstGstCR3 = uCr3;
2637
2638 /* Pass back the translated result. */
2639 *pGCPhysCr3 = Walk.GCPhys;
2640 return VINF_SUCCESS;
2641 }
2642
2643 /* Translation failed. */
2644 *pGCPhysCr3 = NIL_RTGCPHYS;
2645 return rc;
2646 }
2647
2648 /*
2649 * If the nested-guest CR3 has not changed, then the previously
2650 * translated CR3 result (i.e. GCPhysCR3) is passed back.
2651 */
2652 *pGCPhysCr3 = pVCpu->pgm.s.GCPhysCR3;
2653 return VINF_SUCCESS;
2654}
2655
2656# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
2657
2658/**
2659 * Performs and schedules necessary updates following a CR3 load or reload.
2660 *
2661 * This will normally involve mapping the guest PD or nPDPT
2662 *
2663 * @returns VBox status code.
2664 * @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync. This can
2665 * safely be ignored and overridden since the FF will be set too then.
2666 * @param pVCpu The cross context virtual CPU structure.
2667 * @param cr3 The new cr3.
2668 * @param fGlobal Indicates whether this is a global flush or not.
2669 */
2670VMMDECL(int) PGMFlushTLB(PVMCPUCC pVCpu, uint64_t cr3, bool fGlobal)
2671{
2672 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2673
2674 VMCPU_ASSERT_EMT(pVCpu);
2675
2676 /*
2677 * Always flag the necessary updates; necessary for hardware acceleration
2678 */
2679 /** @todo optimize this, it shouldn't always be necessary. */
2680 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2681 if (fGlobal)
2682 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2683
2684 /*
2685 * Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2686 */
2687 RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2688 RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2689#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2690 if ( pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT
2691 && PGMMODE_WITH_PAGING(pVCpu->pgm.s.enmGuestMode))
2692 {
2693 RTGCPHYS GCPhysOut;
2694 int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2695 if (RT_SUCCESS(rc))
2696 GCPhysCR3 = GCPhysOut;
2697 else
2698 {
2699 /* CR3 SLAT translation failed but we try to pretend it
2700 succeeded for the reasons mentioned in PGMHCChangeMode(). */
2701 AssertMsgFailed(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
2702 int const rc2 = pgmGstUnmapCr3(pVCpu);
2703 pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2704 pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2705 return rc2;
2706 }
2707 }
2708#endif
2709
2710 LogFlowFunc(("cr3=%RX64 old=%RX64 fGlobal=%d\n", cr3, GCPhysOldCR3, fGlobal));
2711 int rc = VINF_SUCCESS;
2712 if (GCPhysOldCR3 != GCPhysCR3)
2713 {
2714 uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2715 AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2716 AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2717
2718 pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2719 rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2720 if (RT_LIKELY(rc == VINF_SUCCESS))
2721 { }
2722 else
2723 {
2724 AssertMsg(rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc));
2725 Assert(VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_PGM_SYNC_CR3));
2726 pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped) = false;
2727 pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
2728 pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2729 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_MAP_CR3;
2730 }
2731
2732 if (fGlobal)
2733 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3Global));
2734 else
2735 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3));
2736 }
2737 else
2738 {
2739#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2740 PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
2741 PPGMPOOL const pPool = pVM->pgm.s.CTX_SUFF(pPool);
2742 if (pPool->cDirtyPages)
2743 {
2744 PGM_LOCK_VOID(pVM);
2745 pgmPoolResetDirtyPages(pVM);
2746 PGM_UNLOCK(pVM);
2747 }
2748#endif
2749 if (fGlobal)
2750 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3Global));
2751 else
2752 STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3));
2753
2754 /*
2755 * Flush PAE PDPTEs.
2756 */
2757 if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2758 pgmGstFlushPaePdpes(pVCpu);
2759 }
2760
2761 if (!fGlobal)
2762 IEMTlbInvalidateAll(pVCpu);
2763 else
2764 IEMTlbInvalidateAllGlobal(pVCpu);
2765 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2766 return rc;
2767}
2768
2769
2770/**
2771 * Performs and schedules necessary updates following a CR3 load or reload when
2772 * using nested or extended paging.
2773 *
2774 * This API is an alternative to PGMFlushTLB that avoids actually flushing the
2775 * TLB and triggering a SyncCR3.
2776 *
2777 * This will normally involve mapping the guest PD or nPDPT
2778 *
2779 * @returns VBox status code.
2780 * @retval VINF_SUCCESS.
2781 * @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync (not for nested
2782 * paging modes). This can safely be ignored and overridden since the
2783 * FF will be set too then.
2784 * @param pVCpu The cross context virtual CPU structure.
2785 * @param cr3 The new CR3.
2786 */
2787VMMDECL(int) PGMUpdateCR3(PVMCPUCC pVCpu, uint64_t cr3)
2788{
2789 VMCPU_ASSERT_EMT(pVCpu);
2790
2791 /* We assume we're only called in nested paging mode. */
2792 Assert(pVCpu->CTX_SUFF(pVM)->pgm.s.fNestedPaging || pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2793
2794 /*
2795 * Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2796 */
2797 RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2798 RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2799#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2800 if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2801 {
2802 RTGCPHYS GCPhysOut;
2803 int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2804 if (RT_SUCCESS(rc))
2805 GCPhysCR3 = GCPhysOut;
2806 else
2807 {
2808 /* CR3 SLAT translation failed but we try to pretend it
2809 succeeded for the reasons mentioned in PGMHCChangeMode(). */
2810 Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
2811 int const rc2 = pgmGstUnmapCr3(pVCpu);
2812 pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2813 pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2814 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2815 return rc2;
2816 }
2817 }
2818#endif
2819
2820 LogFlowFunc(("cr3=%RX64 old=%RX64\n", cr3, GCPhysOldCR3));
2821 int rc = VINF_SUCCESS;
2822 if (GCPhysOldCR3 != GCPhysCR3)
2823 {
2824 uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2825 AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2826 AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2827
2828 pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2829 rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2830
2831 AssertRCSuccess(rc); /* Assumes VINF_PGM_SYNC_CR3 doesn't apply to nested paging. */ /** @todo this isn't true for the mac, but we need hw to test/fix this. */
2832 }
2833 /*
2834 * Flush PAE PDPTEs.
2835 */
2836 else if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2837 pgmGstFlushPaePdpes(pVCpu);
2838
2839 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2840 return rc;
2841}
2842
2843
2844/**
2845 * Synchronize the paging structures.
2846 *
2847 * This function is called in response to the VM_FF_PGM_SYNC_CR3 and
2848 * VM_FF_PGM_SYNC_CR3_NONGLOBAL. Those two force action flags are set
2849 * in several places, most importantly whenever the CR3 is loaded.
2850 *
2851 * @returns VBox status code. May return VINF_PGM_SYNC_CR3 in RC/R0.
2852 * @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
2853 * the VMM into guest context.
2854 * @param pVCpu The cross context virtual CPU structure.
2855 * @param cr0 Guest context CR0 register
2856 * @param cr3 Guest context CR3 register
2857 * @param cr4 Guest context CR4 register
2858 * @param fGlobal Including global page directories or not
2859 */
2860VMMDECL(int) PGMSyncCR3(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2861{
2862 VMCPU_ASSERT_EMT(pVCpu);
2863
2864#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2865 /*
2866 * The pool may have pending stuff and even require a return to ring-3 to
2867 * clear the whole thing.
2868 */
2869 int rcPool = pgmPoolSyncCR3(pVCpu);
2870 if (rcPool != VINF_SUCCESS)
2871 return rcPool;
2872#endif
2873
2874 /*
2875 * We might be called when we shouldn't.
2876 *
2877 * The mode switching will ensure that the PD is resynced after every mode
2878 * switch. So, if we find ourselves here when in protected or real mode
2879 * we can safely clear the FF and return immediately.
2880 */
2881 if (pVCpu->pgm.s.enmGuestMode <= PGMMODE_PROTECTED)
2882 {
2883 Assert((cr0 & (X86_CR0_PG | X86_CR0_PE)) != (X86_CR0_PG | X86_CR0_PE));
2884 Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2885 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2886 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2887 return VINF_SUCCESS;
2888 }
2889
2890 /* If global pages are not supported, then all flushes are global. */
2891 if (!(cr4 & X86_CR4_PGE))
2892 fGlobal = true;
2893 LogFlow(("PGMSyncCR3: cr0=%RX64 cr3=%RX64 cr4=%RX64 fGlobal=%d[%d,%d]\n", cr0, cr3, cr4, fGlobal,
2894 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)));
2895
2896 /*
2897 * Check if we need to finish an aborted MapCR3 call (see PGMFlushTLB).
2898 * This should be done before SyncCR3.
2899 */
2900 if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MAP_CR3)
2901 {
2902 pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MAP_CR3;
2903
2904 RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2905 RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2906#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2907 if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2908 {
2909 RTGCPHYS GCPhysOut;
2910 int rc2 = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2911 if (RT_SUCCESS(rc2))
2912 GCPhysCR3 = GCPhysOut;
2913 else
2914 {
2915 /* CR3 SLAT translation failed but we try to pretend it
2916 succeeded for the reasons mentioned in PGMHCChangeMode(). */
2917 AssertMsgFailed(("Failed to translate CR3 %#RX64. rc2=%Rrc\n", cr3, rc2));
2918 pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2919 pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2920 return rc2;
2921 }
2922 }
2923#endif
2924 Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped));
2925 int rc = VINF_SUCCESS;
2926 if (GCPhysOldCR3 != GCPhysCR3)
2927 {
2928 uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2929 AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2930 AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2931 pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2932 rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2933 }
2934
2935 /* Make sure we check for pending pgm pool syncs as we clear VMCPU_FF_PGM_SYNC_CR3 later on! */
2936 if ( rc == VINF_PGM_SYNC_CR3
2937 || (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2938 {
2939 Log(("PGMSyncCR3: pending pgm pool sync after MapCR3!\n"));
2940#ifdef IN_RING3
2941# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2942 rc = pgmPoolSyncCR3(pVCpu);
2943# else
2944 rc = VINF_SUCCESS;
2945# endif
2946#else
2947 if (rc == VINF_PGM_SYNC_CR3)
2948 pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2949 return VINF_PGM_SYNC_CR3;
2950#endif
2951 }
2952 AssertRCReturn(rc, rc);
2953 AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
2954 }
2955
2956 /*
2957 * Let the 'Bth' function do the work and we'll just keep track of the flags.
2958 */
2959 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2960
2961 uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2962 AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2963 AssertReturn(g_aPgmBothModeData[idxBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
2964 int rcSync = g_aPgmBothModeData[idxBth].pfnSyncCR3(pVCpu, cr0, cr3, cr4, fGlobal);
2965
2966 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2967 AssertMsg(rcSync == VINF_SUCCESS || rcSync == VINF_PGM_SYNC_CR3 || RT_FAILURE(rcSync), ("rcSync=%Rrc\n", rcSync));
2968 if (rcSync == VINF_SUCCESS)
2969 {
2970 if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2971 {
2972 /* Go back to ring 3 if a pgm pool sync is again pending. */
2973 return VINF_PGM_SYNC_CR3;
2974 }
2975
2976 if (!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS))
2977 {
2978 Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2979 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2980 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2981 }
2982 }
2983
2984 /*
2985 * Now flush the CR3 (guest context).
2986 */
2987 if (rcSync == VINF_SUCCESS)
2988 PGM_INVL_VCPU_TLBS(pVCpu);
2989 return rcSync;
2990}
2991
2992
2993# ifdef VBOX_STRICT
2994/**
2995 * Asserts that everything related to the guest CR3 is correctly shadowed.
2996 *
2997 * This will call PGMAssertNoMappingConflicts() and PGMAssertHandlerAndFlagsInSync(),
2998 * and assert the correctness of the guest CR3 mapping before asserting that the
2999 * shadow page tables is in sync with the guest page tables.
3000 *
3001 * @returns Number of conflicts.
3002 * @param pVM The cross context VM structure.
3003 * @param pVCpu The cross context virtual CPU structure.
3004 * @param cr3 The current guest CR3 register value.
3005 * @param cr4 The current guest CR4 register value.
3006 */
3007VMMDECL(unsigned) PGMAssertCR3(PVMCC pVM, PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4)
3008{
3009 AssertReturn(pVM->enmTarget == VMTARGET_X86, 0);
3010 STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
3011
3012 uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
3013 AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), -VERR_PGM_MODE_IPE);
3014 AssertReturn(g_aPgmBothModeData[idxBth].pfnAssertCR3, -VERR_PGM_MODE_IPE);
3015
3016 PGM_LOCK_VOID(pVM);
3017 unsigned cErrors = g_aPgmBothModeData[idxBth].pfnAssertCR3(pVCpu, cr3, cr4, 0, ~(RTGCPTR)0);
3018 PGM_UNLOCK(pVM);
3019
3020 STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
3021 return cErrors;
3022}
3023# endif /* VBOX_STRICT */
3024
3025
3026/**
3027 * Called by CPUM or REM when CR0.WP changes to 1.
3028 *
3029 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
3030 * @thread EMT
3031 */
3032VMMDECL(void) PGMCr0WpEnabled(PVMCPUCC pVCpu)
3033{
3034 /*
3035 * Netware WP0+RO+US hack cleanup when WP0 -> WP1.
3036 *
3037 * Use the counter to judge whether there might be pool pages with active
3038 * hacks in them. If there are, we will be running the risk of messing up
3039 * the guest by allowing it to write to read-only pages. Thus, we have to
3040 * clear the page pool ASAP if there is the slightest chance.
3041 */
3042 if (pVCpu->pgm.s.cNetwareWp0Hacks > 0)
3043 {
3044 Assert(pVCpu->CTX_SUFF(pVM)->cCpus == 1);
3045
3046 Log(("PGMCr0WpEnabled: %llu WP0 hacks active - clearing page pool\n", pVCpu->pgm.s.cNetwareWp0Hacks));
3047 pVCpu->pgm.s.cNetwareWp0Hacks = 0;
3048 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
3049 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3050 }
3051}
3052
3053
3054/**
3055 * Updates PGM's copy of the guest's EPT pointer.
3056 *
3057 * @param pVCpu The cross context virtual CPU structure.
3058 * @param uEptPtr The EPT pointer.
3059 *
3060 * @remarks This can be called as part of VM-entry so we might be in the midst of
3061 * switching to VMX non-root mode.
3062 */
3063VMM_INT_DECL(void) PGMSetGuestEptPtr(PVMCPUCC pVCpu, uint64_t uEptPtr)
3064{
3065 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3066 PGM_LOCK_VOID(pVM);
3067 pVCpu->pgm.s.uEptPtr = uEptPtr;
3068 pVCpu->pgm.s.pGstEptPml4R3 = 0;
3069 pVCpu->pgm.s.pGstEptPml4R0 = 0;
3070 PGM_UNLOCK(pVM);
3071}
3072
3073
3074/**
3075 * Maps all the PAE PDPE entries.
3076 *
3077 * @returns VBox status code.
3078 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
3079 * @param paPaePdpes The new PAE PDPE values.
3080 *
3081 * @remarks This function may be invoked during the process of changing the guest
3082 * paging mode to PAE, hence the guest state (CR0, CR4 etc.) may not
3083 * reflect PAE paging just yet.
3084 */
3085VMM_INT_DECL(int) PGMGstMapPaePdpes(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
3086{
3087 Assert(paPaePdpes);
3088 for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
3089 {
3090 X86PDPE const PaePdpe = paPaePdpes[i];
3091
3092 /*
3093 * In some cases (e.g. in SVM with nested paging) the validation of the PAE PDPEs
3094 * are deferred.[1] Also, different situations require different handling of invalid
3095 * PDPE entries. Here we assume the caller has already validated or doesn't require
3096 * validation of the PDPEs.
3097 *
3098 * In the case of nested EPT (i.e. for nested-guests), the PAE PDPEs have been
3099 * validated by the VMX transition.
3100 *
3101 * [1] -- See AMD spec. 15.25.10 "Legacy PAE Mode".
3102 */
3103 if ((PaePdpe.u & (pVCpu->pgm.s.fGstPaeMbzPdpeMask | X86_PDPE_P)) == X86_PDPE_P)
3104 {
3105 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3106 RTHCPTR HCPtr;
3107
3108 RTGCPHYS GCPhys;
3109#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3110 if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
3111 {
3112 PGMPTWALK Walk;
3113 PGMPTWALKGST GstWalk;
3114 RTGCPHYS const GCPhysNested = PaePdpe.u & X86_PDPE_PG_MASK;
3115 int const rc = pgmGstSlatWalk(pVCpu, GCPhysNested, false /* fIsLinearAddrValid */, 0 /* GCPtrNested */,
3116 &Walk, &GstWalk);
3117 if (RT_SUCCESS(rc))
3118 GCPhys = Walk.GCPhys;
3119 else
3120 {
3121 /*
3122 * Second-level address translation of the PAE PDPE has failed but we must -NOT-
3123 * abort and return a failure now. This is because we're called from a Mov CRx
3124 * instruction (or similar operation). Let's just pretend success but flag that
3125 * we need to map this PDPE lazily later.
3126 *
3127 * See Intel spec. 25.3 "Changes to instruction behavior in VMX non-root operation".
3128 * See Intel spec. 28.3.1 "EPT Overview".
3129 */
3130 pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
3131 pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
3132 pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
3133 continue;
3134 }
3135 }
3136 else
3137#endif
3138 {
3139 GCPhys = PGM_A20_APPLY(pVCpu, PaePdpe.u & X86_PDPE_PG_MASK);
3140 }
3141
3142 PGM_LOCK_VOID(pVM);
3143 PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
3144 AssertReturnStmt(pPage, PGM_UNLOCK(pVM), VERR_PGM_INVALID_PDPE_ADDR);
3145 int const rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
3146 PGM_UNLOCK(pVM);
3147 if (RT_SUCCESS(rc))
3148 {
3149#ifdef IN_RING3
3150 pVCpu->pgm.s.apGstPaePDsR3[i] = (PX86PDPAE)HCPtr;
3151 pVCpu->pgm.s.apGstPaePDsR0[i] = NIL_RTR0PTR;
3152#else
3153 pVCpu->pgm.s.apGstPaePDsR3[i] = NIL_RTR3PTR;
3154 pVCpu->pgm.s.apGstPaePDsR0[i] = (PX86PDPAE)HCPtr;
3155#endif
3156 pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
3157 continue;
3158 }
3159 AssertMsgFailed(("PGMPhysMapPaePdpes: rc2=%d GCPhys=%RGp i=%d\n", rc, GCPhys, i));
3160 }
3161 pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
3162 pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
3163 pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
3164 }
3165 return VINF_SUCCESS;
3166}
3167
3168
3169/**
3170 * Validates and maps the PDPT and PAE PDPEs referenced by the given CR3.
3171 *
3172 * @returns VBox status code.
3173 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
3174 * @param cr3 The guest CR3 value.
3175 *
3176 * @remarks This function may be invoked during the process of changing the guest
3177 * paging mode to PAE but the guest state (CR0, CR4 etc.) may not reflect
3178 * PAE paging just yet.
3179 */
3180VMM_INT_DECL(int) PGMGstMapPaePdpesAtCr3(PVMCPUCC pVCpu, uint64_t cr3)
3181{
3182 /*
3183 * Read the page-directory-pointer table (PDPT) at CR3.
3184 */
3185 RTGCPHYS GCPhysCR3 = (cr3 & X86_CR3_PAE_PAGE_MASK);
3186 PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
3187
3188#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3189 if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
3190 {
3191 RTGCPHYS GCPhysOut;
3192 int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
3193 if (RT_SUCCESS(rc))
3194 GCPhysCR3 = GCPhysOut;
3195 else
3196 {
3197 Log(("Failed to load CR3 at %#RX64. rc=%Rrc\n", GCPhysCR3, rc));
3198 return rc;
3199 }
3200 }
3201#endif
3202
3203 RTHCPTR HCPtrGuestCr3;
3204 int rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCr3);
3205 if (RT_SUCCESS(rc))
3206 {
3207 /*
3208 * Validate the page-directory-pointer table entries (PDPE).
3209 */
3210 X86PDPE aPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
3211 memcpy(&aPaePdpes[0], HCPtrGuestCr3, sizeof(aPaePdpes));
3212 if (PGMGstArePaePdpesValid(pVCpu, &aPaePdpes[0]))
3213 {
3214 /*
3215 * Map the PDPT.
3216 * We deliberately don't update PGM's GCPhysCR3 here as it's expected
3217 * that PGMFlushTLB will be called soon and only a change to CR3 then
3218 * will cause the shadow page tables to be updated.
3219 */
3220#ifdef IN_RING3
3221 pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCr3;
3222 pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
3223#else
3224 pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
3225 pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCr3;
3226#endif
3227
3228 /*
3229 * Update CPUM and map the 4 PAE PDPEs.
3230 */
3231 CPUMSetGuestPaePdpes(pVCpu, &aPaePdpes[0]);
3232 rc = PGMGstMapPaePdpes(pVCpu, &aPaePdpes[0]);
3233 if (RT_SUCCESS(rc))
3234 {
3235#ifdef IN_RING3
3236 pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = true;
3237 pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
3238#else
3239 pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
3240 pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = true;
3241#endif
3242 pVCpu->pgm.s.GCPhysPaeCR3 = GCPhysCR3;
3243 }
3244 }
3245 else
3246 rc = VERR_PGM_PAE_PDPE_RSVD;
3247 }
3248 return rc;
3249}
3250
3251
3252/**
3253 * Called whenever CR0 or CR4 in a way which may affect the paging mode.
3254 *
3255 * @returns VBox status code, with the following informational code for
3256 * VM scheduling.
3257 * @retval VINF_SUCCESS if the was no change, or it was successfully dealt with.
3258 * @retval VINF_EM_SUSPEND or VINF_EM_OFF on a fatal runtime error. (R3 only)
3259 *
3260 * @param pVCpu The cross context virtual CPU structure.
3261 * @param cr0 The new cr0.
3262 * @param cr4 The new cr4.
3263 * @param efer The new extended feature enable register.
3264 * @param fForce Whether to force a mode change.
3265 */
3266VMMDECL(int) PGMChangeMode(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr4, uint64_t efer, bool fForce)
3267{
3268 VMCPU_ASSERT_EMT(pVCpu);
3269
3270 /*
3271 * Calc the new guest mode.
3272 *
3273 * Note! We check PG before PE and without requiring PE because of the
3274 * special AMD-V paged real mode (APM vol 2, rev 3.28, 15.9).
3275 */
3276 PGMMODE enmGuestMode;
3277 if (cr0 & X86_CR0_PG)
3278 {
3279 if (!(cr4 & X86_CR4_PAE))
3280 {
3281 bool const fPse = !!(cr4 & X86_CR4_PSE);
3282 if (pVCpu->pgm.s.fGst32BitPageSizeExtension != fPse)
3283 Log(("PGMChangeMode: CR4.PSE %d -> %d\n", pVCpu->pgm.s.fGst32BitPageSizeExtension, fPse));
3284 pVCpu->pgm.s.fGst32BitPageSizeExtension = fPse;
3285 enmGuestMode = PGMMODE_32_BIT;
3286 }
3287 else if (!(efer & MSR_K6_EFER_LME))
3288 {
3289 if (!(efer & MSR_K6_EFER_NXE))
3290 enmGuestMode = PGMMODE_PAE;
3291 else
3292 enmGuestMode = PGMMODE_PAE_NX;
3293 }
3294 else
3295 {
3296 if (!(efer & MSR_K6_EFER_NXE))
3297 enmGuestMode = PGMMODE_AMD64;
3298 else
3299 enmGuestMode = PGMMODE_AMD64_NX;
3300 }
3301 }
3302 else if (!(cr0 & X86_CR0_PE))
3303 enmGuestMode = PGMMODE_REAL;
3304 else
3305 enmGuestMode = PGMMODE_PROTECTED;
3306
3307 /*
3308 * Did it change?
3309 */
3310 if ( !fForce
3311 && pVCpu->pgm.s.enmGuestMode == enmGuestMode)
3312 return VINF_SUCCESS;
3313
3314 /* Flush the TLB */
3315 PGM_INVL_VCPU_TLBS(pVCpu);
3316 return PGMHCChangeMode(pVCpu->CTX_SUFF(pVM), pVCpu, enmGuestMode, fForce);
3317}
3318
3319
3320/**
3321 * Converts a PGMMODE value to a PGM_TYPE_* \#define.
3322 *
3323 * @returns PGM_TYPE_*.
3324 * @param pgmMode The mode value to convert.
3325 */
3326DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
3327{
3328 switch (pgmMode)
3329 {
3330 case PGMMODE_REAL: return PGM_TYPE_REAL;
3331 case PGMMODE_PROTECTED: return PGM_TYPE_PROT;
3332 case PGMMODE_32_BIT: return PGM_TYPE_32BIT;
3333 case PGMMODE_PAE:
3334 case PGMMODE_PAE_NX: return PGM_TYPE_PAE;
3335 case PGMMODE_AMD64:
3336 case PGMMODE_AMD64_NX: return PGM_TYPE_AMD64;
3337 case PGMMODE_NESTED_32BIT: return PGM_TYPE_NESTED_32BIT;
3338 case PGMMODE_NESTED_PAE: return PGM_TYPE_NESTED_PAE;
3339 case PGMMODE_NESTED_AMD64: return PGM_TYPE_NESTED_AMD64;
3340 case PGMMODE_EPT: return PGM_TYPE_EPT;
3341 case PGMMODE_NONE: return PGM_TYPE_NONE;
3342 default:
3343 AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
3344 }
3345}
3346
3347
3348/**
3349 * Calculates the shadow paging mode.
3350 *
3351 * @returns The shadow paging mode.
3352 * @param pVM The cross context VM structure.
3353 * @param enmGuestMode The guest mode.
3354 * @param enmHostMode The host mode.
3355 * @param enmShadowMode The current shadow mode.
3356 */
3357static PGMMODE pgmCalcShadowMode(PVMCC pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode)
3358{
3359 switch (enmGuestMode)
3360 {
3361 case PGMMODE_REAL:
3362 case PGMMODE_PROTECTED:
3363 switch (enmHostMode)
3364 {
3365 case SUPPAGINGMODE_32_BIT:
3366 case SUPPAGINGMODE_32_BIT_GLOBAL:
3367 enmShadowMode = PGMMODE_32_BIT;
3368 break;
3369
3370 case SUPPAGINGMODE_PAE:
3371 case SUPPAGINGMODE_PAE_NX:
3372 case SUPPAGINGMODE_PAE_GLOBAL:
3373 case SUPPAGINGMODE_PAE_GLOBAL_NX:
3374 enmShadowMode = PGMMODE_PAE;
3375 break;
3376
3377 case SUPPAGINGMODE_AMD64:
3378 case SUPPAGINGMODE_AMD64_GLOBAL:
3379 case SUPPAGINGMODE_AMD64_NX:
3380 case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3381 enmShadowMode = PGMMODE_PAE;
3382 break;
3383
3384 default:
3385 AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3386 }
3387 break;
3388
3389 case PGMMODE_32_BIT:
3390 switch (enmHostMode)
3391 {
3392 case SUPPAGINGMODE_32_BIT:
3393 case SUPPAGINGMODE_32_BIT_GLOBAL:
3394 enmShadowMode = PGMMODE_32_BIT;
3395 break;
3396
3397 case SUPPAGINGMODE_PAE:
3398 case SUPPAGINGMODE_PAE_NX:
3399 case SUPPAGINGMODE_PAE_GLOBAL:
3400 case SUPPAGINGMODE_PAE_GLOBAL_NX:
3401 enmShadowMode = PGMMODE_PAE;
3402 break;
3403
3404 case SUPPAGINGMODE_AMD64:
3405 case SUPPAGINGMODE_AMD64_GLOBAL:
3406 case SUPPAGINGMODE_AMD64_NX:
3407 case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3408 enmShadowMode = PGMMODE_PAE;
3409 break;
3410
3411 default:
3412 AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3413 }
3414 break;
3415
3416 case PGMMODE_PAE:
3417 case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
3418 switch (enmHostMode)
3419 {
3420 case SUPPAGINGMODE_32_BIT:
3421 case SUPPAGINGMODE_32_BIT_GLOBAL:
3422 enmShadowMode = PGMMODE_PAE;
3423 break;
3424
3425 case SUPPAGINGMODE_PAE:
3426 case SUPPAGINGMODE_PAE_NX:
3427 case SUPPAGINGMODE_PAE_GLOBAL:
3428 case SUPPAGINGMODE_PAE_GLOBAL_NX:
3429 enmShadowMode = PGMMODE_PAE;
3430 break;
3431
3432 case SUPPAGINGMODE_AMD64:
3433 case SUPPAGINGMODE_AMD64_GLOBAL:
3434 case SUPPAGINGMODE_AMD64_NX:
3435 case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3436 enmShadowMode = PGMMODE_PAE;
3437 break;
3438
3439 default:
3440 AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3441 }
3442 break;
3443
3444 case PGMMODE_AMD64:
3445 case PGMMODE_AMD64_NX:
3446 switch (enmHostMode)
3447 {
3448 case SUPPAGINGMODE_32_BIT:
3449 case SUPPAGINGMODE_32_BIT_GLOBAL:
3450 enmShadowMode = PGMMODE_AMD64;
3451 break;
3452
3453 case SUPPAGINGMODE_PAE:
3454 case SUPPAGINGMODE_PAE_NX:
3455 case SUPPAGINGMODE_PAE_GLOBAL:
3456 case SUPPAGINGMODE_PAE_GLOBAL_NX:
3457 enmShadowMode = PGMMODE_AMD64;
3458 break;
3459
3460 case SUPPAGINGMODE_AMD64:
3461 case SUPPAGINGMODE_AMD64_GLOBAL:
3462 case SUPPAGINGMODE_AMD64_NX:
3463 case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3464 enmShadowMode = PGMMODE_AMD64;
3465 break;
3466
3467 default:
3468 AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3469 }
3470 break;
3471
3472 default:
3473 AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), PGMMODE_INVALID);
3474 }
3475
3476 /*
3477 * Override the shadow mode when NEM, IEM or nested paging is active.
3478 */
3479 if (!VM_IS_HM_ENABLED(pVM))
3480 {
3481 Assert(VM_IS_NEM_ENABLED(pVM) || VM_IS_EXEC_ENGINE_IEM(pVM));
3482 pVM->pgm.s.fNestedPaging = true;
3483 enmShadowMode = PGMMODE_NONE;
3484 }
3485 else
3486 {
3487 bool fNestedPaging = HMIsNestedPagingActive(pVM);
3488 pVM->pgm.s.fNestedPaging = fNestedPaging;
3489 if (fNestedPaging)
3490 {
3491 if (HMIsVmxActive(pVM))
3492 enmShadowMode = PGMMODE_EPT;
3493 else
3494 {
3495 /* The nested SVM paging depends on the host one. */
3496 Assert(HMIsSvmActive(pVM));
3497 if ( enmGuestMode == PGMMODE_AMD64
3498 || enmGuestMode == PGMMODE_AMD64_NX)
3499 enmShadowMode = PGMMODE_NESTED_AMD64;
3500 else
3501 switch (pVM->pgm.s.enmHostMode)
3502 {
3503 case SUPPAGINGMODE_32_BIT:
3504 case SUPPAGINGMODE_32_BIT_GLOBAL:
3505 enmShadowMode = PGMMODE_NESTED_32BIT;
3506 break;
3507
3508 case SUPPAGINGMODE_PAE:
3509 case SUPPAGINGMODE_PAE_GLOBAL:
3510 case SUPPAGINGMODE_PAE_NX:
3511 case SUPPAGINGMODE_PAE_GLOBAL_NX:
3512 enmShadowMode = PGMMODE_NESTED_PAE;
3513 break;
3514
3515 case SUPPAGINGMODE_AMD64:
3516 case SUPPAGINGMODE_AMD64_GLOBAL:
3517 case SUPPAGINGMODE_AMD64_NX:
3518 case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3519 enmShadowMode = PGMMODE_NESTED_AMD64;
3520 break;
3521
3522 default:
3523 AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3524 }
3525 }
3526 }
3527#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3528 else
3529 {
3530 /* Nested paging is a requirement for nested VT-x. */
3531 AssertLogRelMsgReturn(enmGuestMode != PGMMODE_EPT, ("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3532 }
3533#endif
3534 }
3535
3536 return enmShadowMode;
3537}
3538
3539#endif /* VBOX_VMM_TARGET_X86 */
3540
3541/**
3542 * Performs the actual mode change.
3543 * This is called by PGMChangeMode and pgmR3InitPaging().
3544 *
3545 * @returns VBox status code. May suspend or power off the VM on error, but this
3546 * will trigger using FFs and not informational status codes.
3547 *
3548 * @param pVM The cross context VM structure.
3549 * @param pVCpu The cross context virtual CPU structure.
3550 * @param enmGuestMode The new guest mode. This is assumed to be different from
3551 * the current mode.
3552 * @param fForce Whether to force a shadow paging mode change.
3553 */
3554VMM_INT_DECL(int) PGMHCChangeMode(PVMCC pVM, PVMCPUCC pVCpu, PGMMODE enmGuestMode, bool fForce)
3555{
3556 Log(("PGMHCChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
3557 STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
3558
3559#ifdef VBOX_VMM_TARGET_X86
3560 /*
3561 * Calc the shadow mode and switcher.
3562 */
3563 PGMMODE const enmShadowMode = pgmCalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode);
3564 bool const fShadowModeChanged = enmShadowMode != pVCpu->pgm.s.enmShadowMode || fForce;
3565
3566 /*
3567 * Exit old mode(s).
3568 */
3569 /* shadow */
3570 if (fShadowModeChanged)
3571 {
3572 LogFlow(("PGMHCChangeMode: Shadow mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
3573 uintptr_t idxOldShw = pVCpu->pgm.s.idxShadowModeData;
3574 if ( idxOldShw < RT_ELEMENTS(g_aPgmShadowModeData)
3575 && g_aPgmShadowModeData[idxOldShw].pfnExit)
3576 {
3577 int rc = g_aPgmShadowModeData[idxOldShw].pfnExit(pVCpu);
3578 AssertMsgRCReturn(rc, ("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc), rc);
3579 }
3580 }
3581 else
3582 LogFlow(("PGMHCChangeMode: Shadow mode remains: %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
3583
3584 /* guest */
3585 uintptr_t const idxOldGst = pVCpu->pgm.s.idxGuestModeData;
3586 if ( idxOldGst < RT_ELEMENTS(g_aPgmGuestModeData)
3587 && g_aPgmGuestModeData[idxOldGst].pfnExit)
3588 {
3589 int rc = g_aPgmGuestModeData[idxOldGst].pfnExit(pVCpu);
3590 AssertMsgReturn(RT_SUCCESS(rc), ("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc), rc);
3591 }
3592 pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
3593 pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
3594 pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
3595 Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped));
3596
3597 /*
3598 * Change the paging mode data indexes.
3599 */
3600 uintptr_t idxNewGst = pVCpu->pgm.s.idxGuestModeData = pgmModeToType(enmGuestMode);
3601 AssertReturn(idxNewGst < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
3602 AssertReturn(g_aPgmGuestModeData[idxNewGst].uType == idxNewGst, VERR_PGM_MODE_IPE);
3603 AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPage, VERR_PGM_MODE_IPE);
3604 AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnModifyPage, VERR_PGM_MODE_IPE);
3605 AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnExit, VERR_PGM_MODE_IPE);
3606 AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnEnter, VERR_PGM_MODE_IPE);
3607# ifdef IN_RING3
3608 AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnRelocate, VERR_PGM_MODE_IPE);
3609# endif
3610
3611 uintptr_t const idxNewShw = pVCpu->pgm.s.idxShadowModeData = pgmModeToType(enmShadowMode);
3612 AssertReturn(idxNewShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
3613 AssertReturn(g_aPgmShadowModeData[idxNewShw].uType == idxNewShw, VERR_PGM_MODE_IPE);
3614 AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnGetPage, VERR_PGM_MODE_IPE);
3615 AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnModifyPage, VERR_PGM_MODE_IPE);
3616 AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnExit, VERR_PGM_MODE_IPE);
3617 AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnEnter, VERR_PGM_MODE_IPE);
3618# ifdef IN_RING3
3619 AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnRelocate, VERR_PGM_MODE_IPE);
3620# endif
3621
3622 uintptr_t const idxNewBth = pVCpu->pgm.s.idxBothModeData = (idxNewShw - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END + idxNewGst;
3623 AssertReturn(g_aPgmBothModeData[idxNewBth].uShwType == idxNewShw, VERR_PGM_MODE_IPE);
3624 AssertReturn(g_aPgmBothModeData[idxNewBth].uGstType == idxNewGst, VERR_PGM_MODE_IPE);
3625 AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnInvalidatePage, VERR_PGM_MODE_IPE);
3626 AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
3627 AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
3628 AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnMapCR3, VERR_PGM_MODE_IPE);
3629 AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
3630 AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnEnter, VERR_PGM_MODE_IPE);
3631# ifdef VBOX_STRICT
3632 AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnAssertCR3, VERR_PGM_MODE_IPE);
3633# endif
3634
3635 /*
3636 * Determine SLAT mode -before- entering the new shadow mode!
3637 */
3638 pVCpu->pgm.s.enmGuestSlatMode = !CPUMIsGuestVmxEptPagingEnabled(pVCpu) ? PGMSLAT_DIRECT : PGMSLAT_EPT;
3639
3640 /*
3641 * Enter new shadow mode (if changed).
3642 */
3643 if (fShadowModeChanged)
3644 {
3645 pVCpu->pgm.s.enmShadowMode = enmShadowMode;
3646 int rc = g_aPgmShadowModeData[idxNewShw].pfnEnter(pVCpu);
3647 AssertLogRelMsgRCReturnStmt(rc, ("Entering enmShadowMode=%s failed: %Rrc\n", PGMGetModeName(enmShadowMode), rc),
3648 pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID, rc);
3649 }
3650
3651 /*
3652 * Always flag the necessary updates
3653 */
3654 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3655
3656 /*
3657 * Enter the new guest and shadow+guest modes.
3658 */
3659 /* Calc the new CR3 value. */
3660 RTGCPHYS GCPhysCR3;
3661 switch (enmGuestMode)
3662 {
3663 case PGMMODE_REAL:
3664 case PGMMODE_PROTECTED:
3665 GCPhysCR3 = NIL_RTGCPHYS;
3666 break;
3667
3668 case PGMMODE_32_BIT:
3669 GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
3670 break;
3671
3672 case PGMMODE_PAE_NX:
3673 case PGMMODE_PAE:
3674 if (!pVM->cpum.ro.GuestFeatures.fPae)
3675# ifdef IN_RING3 /** @todo r=bird: wrong place, probably hasn't really worked for a while. */
3676 return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
3677 N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (System/Processor)"));
3678# else
3679 AssertLogRelMsgFailedReturn(("enmGuestMode=%s - Try enable PAE for the guest!\n", PGMGetModeName(enmGuestMode)), VERR_PGM_MODE_IPE);
3680
3681# endif
3682 GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
3683 break;
3684
3685# ifdef VBOX_WITH_64_BITS_GUESTS
3686 case PGMMODE_AMD64_NX:
3687 case PGMMODE_AMD64:
3688 GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_AMD64_PAGE_MASK;
3689 break;
3690# endif
3691 default:
3692 AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), VERR_PGM_MODE_IPE);
3693 }
3694
3695# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3696 /*
3697 * If a nested-guest is using EPT paging:
3698 * - Update the second-level address translation (SLAT) mode.
3699 * - Indicate that the CR3 is nested-guest physical address.
3700 */
3701 if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
3702 {
3703 if (PGMMODE_WITH_PAGING(enmGuestMode))
3704 {
3705 /*
3706 * Translate CR3 to its guest-physical address.
3707 * We don't use pgmGstSlatTranslateCr3() here as we want to update GCPhysNstGstCR3 -after-
3708 * switching modes to keep it consistent with how GCPhysCR3 is updated.
3709 */
3710 PGMPTWALK Walk;
3711 PGMPTWALKGST GstWalk;
3712 int const rc = pgmGstSlatWalk(pVCpu, GCPhysCR3, false /* fIsLinearAddrValid */, 0 /* GCPtrNested */, &Walk,
3713 &GstWalk);
3714 if (RT_SUCCESS(rc))
3715 { /* likely */ }
3716 else
3717 {
3718 /*
3719 * SLAT failed but we avoid reporting this to the caller because the caller
3720 * is not supposed to fail. The only time the caller needs to indicate a
3721 * failure to software is when PAE paging is used by the nested-guest, but
3722 * we handle the PAE case separately (e.g., see VMX transition in IEM).
3723 * In all other cases, the failure will be indicated when CR3 tries to be
3724 * translated on the next linear-address memory access.
3725 * See Intel spec. 27.2.1 "EPT Overview".
3726 */
3727 Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", GCPhysCR3, rc));
3728
3729 /* Trying to coax PGM to succeed for the time being... */
3730 Assert(pVCpu->pgm.s.GCPhysCR3 == NIL_RTGCPHYS);
3731 pVCpu->pgm.s.GCPhysNstGstCR3 = GCPhysCR3;
3732 pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3733 HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3734 return VINF_SUCCESS;
3735 }
3736 pVCpu->pgm.s.GCPhysNstGstCR3 = GCPhysCR3;
3737 GCPhysCR3 = Walk.GCPhys & X86_CR3_EPT_PAGE_MASK;
3738 }
3739 }
3740 else
3741 Assert(pVCpu->pgm.s.GCPhysNstGstCR3 == NIL_RTGCPHYS);
3742# endif
3743
3744 /*
3745 * Enter the new guest mode.
3746 */
3747 pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3748 int rc = g_aPgmGuestModeData[idxNewGst].pfnEnter(pVCpu, GCPhysCR3);
3749 int rc2 = g_aPgmBothModeData[idxNewBth].pfnEnter(pVCpu, GCPhysCR3);
3750
3751 /* Set the new guest CR3 (and nested-guest CR3). */
3752 pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
3753
3754 /* status codes. */
3755 AssertRC(rc);
3756 AssertRC(rc2);
3757 if (RT_SUCCESS(rc))
3758 {
3759 rc = rc2;
3760 if (RT_SUCCESS(rc)) /* no informational status codes. */
3761 rc = VINF_SUCCESS;
3762 }
3763
3764 /*
3765 * Notify HM.
3766 */
3767 HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3768 return rc;
3769
3770#elif defined(VBOX_VMM_TARGET_ARMV8)
3771 //AssertReleaseFailed(); /** @todo Called by the PGM saved state code. */
3772 RT_NOREF(pVM, pVCpu, enmGuestMode, fForce);
3773 return VINF_SUCCESS;
3774
3775#else
3776# error "port me"
3777#endif
3778}
3779
3780
3781/**
3782 * Get mode name.
3783 *
3784 * @returns read-only name string.
3785 * @param enmMode The mode which name is desired.
3786 */
3787VMMDECL(const char *) PGMGetModeName(PGMMODE enmMode)
3788{
3789 switch (enmMode)
3790 {
3791 case PGMMODE_REAL: return "Real";
3792 case PGMMODE_PROTECTED: return "Protected";
3793 case PGMMODE_32_BIT: return "32-bit";
3794 case PGMMODE_PAE: return "PAE";
3795 case PGMMODE_PAE_NX: return "PAE+NX";
3796 case PGMMODE_AMD64: return "AMD64";
3797 case PGMMODE_AMD64_NX: return "AMD64+NX";
3798 case PGMMODE_NESTED_32BIT: return "Nested-32";
3799 case PGMMODE_NESTED_PAE: return "Nested-PAE";
3800 case PGMMODE_NESTED_AMD64: return "Nested-AMD64";
3801 case PGMMODE_EPT: return "EPT";
3802 case PGMMODE_NONE: return "None";
3803 case PGMMODE_VMSA_V8_32: return "VMSAv8-32";
3804 case PGMMODE_VMSA_V8_64: return "VMSAv8-64";
3805
3806 case PGMMODE_INVALID:
3807 case PGMMODE_MAX:
3808 case PGMMODE_32BIT_HACK:
3809 break;
3810 }
3811 return "unknown mode value";
3812}
3813
3814
3815/**
3816 * Gets the current guest paging mode.
3817 *
3818 * If you just need the CPU mode (real/protected/long), use CPUMGetGuestMode().
3819 *
3820 * @returns The current paging mode.
3821 * @param pVCpu The cross context virtual CPU structure.
3822 */
3823VMMDECL(PGMMODE) PGMGetGuestMode(PVMCPU pVCpu)
3824{
3825 return pVCpu->pgm.s.enmGuestMode;
3826}
3827
3828
3829/**
3830 * Gets the current shadow paging mode.
3831 *
3832 * @returns The current paging mode.
3833 * @param pVCpu The cross context virtual CPU structure.
3834 */
3835VMMDECL(PGMMODE) PGMGetShadowMode(PVMCPU pVCpu)
3836{
3837#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
3838 return pVCpu->pgm.s.enmShadowMode;
3839#else
3840 RT_NOREF(pVCpu);
3841 return PGMMODE_NONE;
3842#endif
3843}
3844
3845#ifdef VBOX_VMM_TARGET_X86
3846
3847/**
3848 * Gets the current host paging mode.
3849 *
3850 * @returns The current paging mode.
3851 * @param pVM The cross context VM structure.
3852 */
3853VMMDECL(PGMMODE) PGMGetHostMode(PVM pVM)
3854{
3855 switch (pVM->pgm.s.enmHostMode)
3856 {
3857 case SUPPAGINGMODE_32_BIT:
3858 case SUPPAGINGMODE_32_BIT_GLOBAL:
3859 return PGMMODE_32_BIT;
3860
3861 case SUPPAGINGMODE_PAE:
3862 case SUPPAGINGMODE_PAE_GLOBAL:
3863 return PGMMODE_PAE;
3864
3865 case SUPPAGINGMODE_PAE_NX:
3866 case SUPPAGINGMODE_PAE_GLOBAL_NX:
3867 return PGMMODE_PAE_NX;
3868
3869 case SUPPAGINGMODE_AMD64:
3870 case SUPPAGINGMODE_AMD64_GLOBAL:
3871 return PGMMODE_AMD64;
3872
3873 case SUPPAGINGMODE_AMD64_NX:
3874 case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3875 return PGMMODE_AMD64_NX;
3876
3877 default: AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode)); break;
3878 }
3879
3880 return PGMMODE_INVALID;
3881}
3882
3883
3884# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3885/**
3886 * Gets the SLAT mode name.
3887 *
3888 * @returns The read-only SLAT mode descriptive string.
3889 * @param enmSlatMode The SLAT mode value.
3890 */
3891VMM_INT_DECL(const char *) PGMGetSlatModeName(PGMSLAT enmSlatMode)
3892{
3893 switch (enmSlatMode)
3894 {
3895 case PGMSLAT_DIRECT: return "Direct";
3896 case PGMSLAT_EPT: return "EPT";
3897 case PGMSLAT_32BIT: return "32-bit";
3898 case PGMSLAT_PAE: return "PAE";
3899 case PGMSLAT_AMD64: return "AMD64";
3900 default: return "Unknown";
3901 }
3902}
3903# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
3904
3905
3906/**
3907 * Notification from CPUM that the EFER.NXE bit has changed.
3908 *
3909 * @param pVCpu The cross context virtual CPU structure of the CPU for
3910 * which EFER changed.
3911 * @param fNxe The new NXE state.
3912 */
3913VMM_INT_DECL(void) PGMNotifyNxeChanged(PVMCPU pVCpu, bool fNxe)
3914{
3915/** @todo VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); */
3916 Log(("PGMNotifyNxeChanged: fNxe=%RTbool\n", fNxe));
3917
3918 pVCpu->pgm.s.fNoExecuteEnabled = fNxe;
3919 if (fNxe)
3920 {
3921 /*pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A */
3922 pVCpu->pgm.s.fGstPaeMbzPteMask &= ~X86_PTE_PAE_NX;
3923 pVCpu->pgm.s.fGstPaeMbzPdeMask &= ~X86_PDE_PAE_NX;
3924 pVCpu->pgm.s.fGstPaeMbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3925 /*pVCpu->pgm.s.fGstPaeMbzPdpeMask - N/A */
3926 pVCpu->pgm.s.fGstAmd64MbzPteMask &= ~X86_PTE_PAE_NX;
3927 pVCpu->pgm.s.fGstAmd64MbzPdeMask &= ~X86_PDE_PAE_NX;
3928 pVCpu->pgm.s.fGstAmd64MbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3929 pVCpu->pgm.s.fGstAmd64MbzPdpeMask &= ~X86_PDPE_LM_NX;
3930 pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask &= ~X86_PDPE_LM_NX;
3931 pVCpu->pgm.s.fGstAmd64MbzPml4eMask &= ~X86_PML4E_NX;
3932
3933 pVCpu->pgm.s.fGst64ShadowedPteMask |= X86_PTE_PAE_NX;
3934 pVCpu->pgm.s.fGst64ShadowedPdeMask |= X86_PDE_PAE_NX;
3935 pVCpu->pgm.s.fGst64ShadowedBigPdeMask |= X86_PDE2M_PAE_NX;
3936 pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask |= X86_PDE2M_PAE_NX;
3937 pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask |= X86_PDPE_LM_NX;
3938 pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask |= X86_PML4E_NX;
3939 }
3940 else
3941 {
3942 /*pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A */
3943 pVCpu->pgm.s.fGstPaeMbzPteMask |= X86_PTE_PAE_NX;
3944 pVCpu->pgm.s.fGstPaeMbzPdeMask |= X86_PDE_PAE_NX;
3945 pVCpu->pgm.s.fGstPaeMbzBigPdeMask |= X86_PDE2M_PAE_NX;
3946 /*pVCpu->pgm.s.fGstPaeMbzPdpeMask -N/A */
3947 pVCpu->pgm.s.fGstAmd64MbzPteMask |= X86_PTE_PAE_NX;
3948 pVCpu->pgm.s.fGstAmd64MbzPdeMask |= X86_PDE_PAE_NX;
3949 pVCpu->pgm.s.fGstAmd64MbzBigPdeMask |= X86_PDE2M_PAE_NX;
3950 pVCpu->pgm.s.fGstAmd64MbzPdpeMask |= X86_PDPE_LM_NX;
3951 pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask |= X86_PDPE_LM_NX;
3952 pVCpu->pgm.s.fGstAmd64MbzPml4eMask |= X86_PML4E_NX;
3953
3954 pVCpu->pgm.s.fGst64ShadowedPteMask &= ~X86_PTE_PAE_NX;
3955 pVCpu->pgm.s.fGst64ShadowedPdeMask &= ~X86_PDE_PAE_NX;
3956 pVCpu->pgm.s.fGst64ShadowedBigPdeMask &= ~X86_PDE2M_PAE_NX;
3957 pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask &= ~X86_PDE2M_PAE_NX;
3958 pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask &= ~X86_PDPE_LM_NX;
3959 pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask &= ~X86_PML4E_NX;
3960 }
3961}
3962
3963#endif /* VBOX_VMM_TARGET_X86 */
3964
3965/**
3966 * Check if any pgm pool pages are marked dirty (not monitored)
3967 *
3968 * @returns bool locked/not locked
3969 * @param pVM The cross context VM structure.
3970 */
3971VMMDECL(bool) PGMHasDirtyPages(PVM pVM)
3972{
3973#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
3974 return pVM->pgm.s.CTX_SUFF(pPool)->cDirtyPages != 0;
3975#else
3976 RT_NOREF(pVM);
3977 return false;
3978#endif
3979}
3980
3981
3982/**
3983 * Enable or disable large page usage
3984 *
3985 * @returns VBox status code.
3986 * @param pVM The cross context VM structure.
3987 * @param fUseLargePages Use/not use large pages
3988 */
3989VMMDECL(int) PGMSetLargePageUsage(PVMCC pVM, bool fUseLargePages)
3990{
3991 VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
3992
3993 pVM->pgm.s.fUseLargePages = fUseLargePages;
3994 return VINF_SUCCESS;
3995}
3996
3997
3998/**
3999 * Check if this VCPU currently owns the PGM lock.
4000 *
4001 * @returns bool owner/not owner
4002 * @param pVM The cross context VM structure.
4003 */
4004VMMDECL(bool) PGMIsLockOwner(PVMCC pVM)
4005{
4006 return PDMCritSectIsOwner(pVM, &pVM->pgm.s.CritSectX);
4007}
4008
4009
4010/**
4011 * Acquire the PGM lock.
4012 *
4013 * @returns VBox status code
4014 * @param pVM The cross context VM structure.
4015 * @param fVoid Set if the caller cannot handle failure returns.
4016 * @param SRC_POS The source position of the caller (RT_SRC_POS).
4017 */
4018#if defined(VBOX_STRICT) || defined(DOXYGEN_RUNNING)
4019int pgmLockDebug(PVMCC pVM, bool fVoid, RT_SRC_POS_DECL)
4020#else
4021int pgmLock(PVMCC pVM, bool fVoid)
4022#endif
4023{
4024#if defined(VBOX_STRICT)
4025 int rc = PDMCritSectEnterDebug(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS, (uintptr_t)ASMReturnAddress(), RT_SRC_POS_ARGS);
4026#else
4027 int rc = PDMCritSectEnter(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS);
4028#endif
4029 if (RT_SUCCESS(rc))
4030 return rc;
4031 if (fVoid)
4032 PDM_CRITSECT_RELEASE_ASSERT_RC(pVM, &pVM->pgm.s.CritSectX, rc);
4033 else
4034 AssertRC(rc);
4035 return rc;
4036}
4037
4038
4039/**
4040 * Release the PGM lock.
4041 *
4042 * @param pVM The cross context VM structure.
4043 */
4044void pgmUnlock(PVMCC pVM)
4045{
4046 uint32_t cDeprecatedPageLocks = pVM->pgm.s.cDeprecatedPageLocks;
4047 pVM->pgm.s.cDeprecatedPageLocks = 0;
4048 int rc = PDMCritSectLeave(pVM, &pVM->pgm.s.CritSectX);
4049 if (rc == VINF_SEM_NESTED)
4050 pVM->pgm.s.cDeprecatedPageLocks = cDeprecatedPageLocks;
4051}
4052
4053
4054#if !defined(IN_R0) || defined(LOG_ENABLED)
4055
4056/** Format handler for PGMPAGE.
4057 * @copydoc FNRTSTRFORMATTYPE */
4058static DECLCALLBACK(size_t) pgmFormatTypeHandlerPage(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
4059 const char *pszType, void const *pvValue,
4060 int cchWidth, int cchPrecision, unsigned fFlags,
4061 void *pvUser)
4062{
4063 size_t cch;
4064 PCPGMPAGE pPage = (PCPGMPAGE)pvValue;
4065 if (RT_VALID_PTR(pPage))
4066 {
4067 char szTmp[64+80];
4068
4069 cch = 0;
4070
4071 /* The single char state stuff. */
4072 static const char s_achPageStates[4] = { 'Z', 'A', 'W', 'S' };
4073 szTmp[cch++] = s_achPageStates[PGM_PAGE_GET_STATE_NA(pPage)];
4074
4075# define IS_PART_INCLUDED(lvl) ( !(fFlags & RTSTR_F_PRECISION) || cchPrecision == (lvl) || cchPrecision >= (lvl)+10 )
4076 if (IS_PART_INCLUDED(5))
4077 {
4078 static const char s_achHandlerStates[4*2] = { '-', 't', 'w', 'a' , '_', 'T', 'W', 'A' };
4079 szTmp[cch++] = s_achHandlerStates[ PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)
4080 | ((uint8_t)PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage) << 2)];
4081 }
4082
4083 /* The type. */
4084 if (IS_PART_INCLUDED(4))
4085 {
4086 szTmp[cch++] = ':';
4087 static const char s_achPageTypes[8][4] = { "INV", "RAM", "MI2", "M2A", "SHA", "ROM", "MIO", "BAD" };
4088 szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][0];
4089 szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][1];
4090 szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][2];
4091 }
4092
4093 /* The numbers. */
4094 if (IS_PART_INCLUDED(3))
4095 {
4096 szTmp[cch++] = ':';
4097 cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_HCPHYS_NA(pPage), 16, 12, 0, RTSTR_F_ZEROPAD | RTSTR_F_64BIT);
4098 }
4099
4100 if (IS_PART_INCLUDED(2))
4101 {
4102 szTmp[cch++] = ':';
4103 cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_PAGEID(pPage), 16, 7, 0, RTSTR_F_ZEROPAD | RTSTR_F_32BIT);
4104 }
4105
4106# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
4107 if (IS_PART_INCLUDED(6))
4108 {
4109 szTmp[cch++] = ':';
4110 static const char s_achRefs[4] = { '-', 'U', '!', 'L' };
4111 szTmp[cch++] = s_achRefs[PGM_PAGE_GET_TD_CREFS_NA(pPage)];
4112 cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_TD_IDX_NA(pPage), 16, 4, 0, RTSTR_F_ZEROPAD | RTSTR_F_16BIT);
4113 }
4114# endif
4115# undef IS_PART_INCLUDED
4116
4117 cch = pfnOutput(pvArgOutput, szTmp, cch);
4118#if 0
4119 size_t cch2 = 0;
4120 szTmp[cch2++] = '(';
4121 cch2 += RTStrFormatNumber(&szTmp[cch2], (uintptr_t)pPage, 16, 18, 0, RTSTR_F_SPECIAL | RTSTR_F_ZEROPAD | RTSTR_F_64BIT);
4122 szTmp[cch2++] = ')';
4123 szTmp[cch2] = '\0';
4124 cch += pfnOutput(pvArgOutput, szTmp, cch2);
4125#endif
4126 }
4127 else
4128 cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmpage-ptr>"));
4129 NOREF(pszType); NOREF(cchWidth); NOREF(pvUser);
4130 return cch;
4131}
4132
4133
4134/** Format handler for PGMRAMRANGE.
4135 * @copydoc FNRTSTRFORMATTYPE */
4136static DECLCALLBACK(size_t) pgmFormatTypeHandlerRamRange(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
4137 const char *pszType, void const *pvValue,
4138 int cchWidth, int cchPrecision, unsigned fFlags,
4139 void *pvUser)
4140{
4141 size_t cch;
4142 PGMRAMRANGE const *pRam = (PGMRAMRANGE const *)pvValue;
4143 if (RT_VALID_PTR(pRam))
4144 {
4145 char szTmp[80];
4146 cch = RTStrPrintf(szTmp, sizeof(szTmp), "%RGp-%RGp", pRam->GCPhys, pRam->GCPhysLast);
4147 cch = pfnOutput(pvArgOutput, szTmp, cch);
4148 }
4149 else
4150 cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmramrange-ptr>"));
4151 NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(pvUser); NOREF(fFlags);
4152 return cch;
4153}
4154
4155/** Format type andlers to be registered/deregistered. */
4156static const struct
4157{
4158 char szType[24];
4159 PFNRTSTRFORMATTYPE pfnHandler;
4160} g_aPgmFormatTypes[] =
4161{
4162 { "pgmpage", pgmFormatTypeHandlerPage },
4163 { "pgmramrange", pgmFormatTypeHandlerRamRange }
4164};
4165
4166#endif /* !IN_R0 || LOG_ENABLED */
4167
4168/**
4169 * Registers the global string format types.
4170 *
4171 * This should be called at module load time or in some other manner that ensure
4172 * that it's called exactly one time.
4173 *
4174 * @returns IPRT status code on RTStrFormatTypeRegister failure.
4175 */
4176VMMDECL(int) PGMRegisterStringFormatTypes(void)
4177{
4178#if !defined(IN_R0) || defined(LOG_ENABLED)
4179 int rc = VINF_SUCCESS;
4180 unsigned i;
4181 for (i = 0; RT_SUCCESS(rc) && i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
4182 {
4183 rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
4184# ifdef IN_RING0
4185 if (rc == VERR_ALREADY_EXISTS)
4186 {
4187 /* in case of cleanup failure in ring-0 */
4188 RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
4189 rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
4190 }
4191# endif
4192 }
4193 if (RT_FAILURE(rc))
4194 while (i-- > 0)
4195 RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
4196
4197 return rc;
4198#else
4199 return VINF_SUCCESS;
4200#endif
4201}
4202
4203
4204/**
4205 * Deregisters the global string format types.
4206 *
4207 * This should be called at module unload time or in some other manner that
4208 * ensure that it's called exactly one time.
4209 */
4210VMMDECL(void) PGMDeregisterStringFormatTypes(void)
4211{
4212#if !defined(IN_R0) || defined(LOG_ENABLED)
4213 for (unsigned i = 0; i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
4214 RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
4215#endif
4216}
4217
4218#ifdef PGM_WITH_PAGE_ZEROING_DETECTION
4219# ifndef VBOX_VMM_TARGET_X86
4220# error "misconfig: PGM_WITH_PAGE_ZEROING_DETECTION not implemented for ARM guests"
4221# endif
4222
4223/**
4224 * Helper for checking whether XMM0 is zero, possibly retriving external state.
4225 */
4226static bool pgmHandlePageZeroingIsXmm0Zero(PVMCPUCC pVCpu, PCPUMCTX pCtx)
4227{
4228 if (pCtx->fExtrn & CPUMCTX_EXTRN_SSE_AVX)
4229 {
4230 int rc = CPUMImportGuestStateOnDemand(pVCpu, CPUMCTX_EXTRN_SSE_AVX);
4231 AssertRCReturn(rc, false);
4232 }
4233 return pCtx->XState.x87.aXMM[0].au64[0] == 0
4234 && pCtx->XState.x87.aXMM[0].au64[1] == 0
4235 && pCtx->XState.x87.aXMM[0].au64[2] == 0
4236 && pCtx->XState.x87.aXMM[0].au64[3] == 0;
4237}
4238
4239
4240/**
4241 * Helper for comparing opcode bytes.
4242 */
4243static bool pgmHandlePageZeroingMatchOpcodes(PVMCPUCC pVCpu, PCPUMCTX pCtx, uint8_t const *pbOpcodes, uint32_t cbOpcodes)
4244{
4245 uint8_t abTmp[64];
4246 AssertMsgReturn(cbOpcodes <= sizeof(abTmp), ("cbOpcodes=%#x\n", cbOpcodes), false);
4247 int rc = PGMPhysSimpleReadGCPtr(pVCpu, abTmp, pCtx->rip + pCtx->cs.u64Base, cbOpcodes);
4248 if (RT_SUCCESS(rc))
4249 return memcmp(abTmp, pbOpcodes, cbOpcodes) == 0;
4250 return false;
4251}
4252
4253
4254/**
4255 * Called on faults on ZERO pages to check if the guest is trying to zero it.
4256 *
4257 * Since it's a waste of time to zero a ZERO page and it will cause an
4258 * unnecessary page allocation, we'd like to detect and avoid this.
4259 * If any known page zeroing code is detected, this function will update the CPU
4260 * state to pretend the page was zeroed by the code.
4261 *
4262 * @returns true if page zeroing code was detected and CPU state updated to skip
4263 * the code.
4264 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
4265 * @param pCtx The guest register context.
4266 */
4267static bool pgmHandlePageZeroingCode(PVMCPUCC pVCpu, PCPUMCTX pCtx)
4268{
4269 CPUMCTX_ASSERT_NOT_EXTRN(pCtx, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER);
4270
4271 /*
4272 * Sort by mode first.
4273 */
4274 if (CPUMIsGuestInLongModeEx(pCtx))
4275 {
4276 if (CPUMIsGuestIn64BitCodeEx(pCtx))
4277 {
4278 /*
4279 * 64-bit code.
4280 */
4281 Log9(("pgmHandlePageZeroingCode: not page zeroing - 64-bit\n"));
4282 }
4283 else if (pCtx->cs.Attr.n.u1DefBig)
4284 Log9(("pgmHandlePageZeroingCode: not page zeroing - 32-bit lm\n"));
4285 else
4286 Log9(("pgmHandlePageZeroingCode: not page zeroing - 16-bit lm\n"));
4287 }
4288 else if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
4289 {
4290 if (pCtx->cs.Attr.n.u1DefBig)
4291 {
4292 /*
4293 * 32-bit paged protected mode code.
4294 */
4295 CPUMCTX_ASSERT_NOT_EXTRN(pCtx, CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RBX
4296 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RBP | CPUMCTX_EXTRN_RSI | CPUMCTX_EXTRN_RDI
4297 | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
4298
4299 /* 1. Generic 'rep stosd' detection. */
4300 static uint8_t const s_abRepStosD[] = { 0xf3, 0xab };
4301 if ( pCtx->eax == 0
4302 && pCtx->ecx == X86_PAGE_SIZE / 4
4303 && !(pCtx->edi & X86_PAGE_OFFSET_MASK)
4304 && pgmHandlePageZeroingMatchOpcodes(pVCpu, pCtx, s_abRepStosD, sizeof(s_abRepStosD)))
4305 {
4306 pCtx->ecx = 0;
4307 pCtx->edi += X86_PAGE_SIZE;
4308 Log9(("pgmHandlePageZeroingCode: REP STOSD: eip=%RX32 -> %RX32\n", pCtx->eip, pCtx->eip + sizeof(s_abRepStosD)));
4309 pCtx->eip += sizeof(s_abRepStosD);
4310 return true;
4311 }
4312
4313 /* 2. Windows 2000 sp4 KiXMMIZeroPageNoSave loop code: */
4314 static uint8_t const s_abW2kSp4XmmZero[] =
4315 {
4316 0x0f, 0x2b, 0x01,
4317 0x0f, 0x2b, 0x41, 0x10,
4318 0x0f, 0x2b, 0x41, 0x20,
4319 0x0f, 0x2b, 0x41, 0x30,
4320 0x83, 0xc1, 0x40,
4321 0x48,
4322 0x75, 0xeb,
4323 };
4324 if ( pCtx->eax == 64
4325 && !(pCtx->ecx & X86_PAGE_OFFSET_MASK)
4326 && pgmHandlePageZeroingMatchOpcodes(pVCpu, pCtx, s_abW2kSp4XmmZero, sizeof(s_abW2kSp4XmmZero))
4327 && pgmHandlePageZeroingIsXmm0Zero(pVCpu, pCtx))
4328 {
4329 pCtx->eax = 1;
4330 pCtx->ecx += X86_PAGE_SIZE;
4331 Log9(("pgmHandlePageZeroingCode: w2k sp4 xmm: eip=%RX32 -> %RX32\n",
4332 pCtx->eip, pCtx->eip + sizeof(s_abW2kSp4XmmZero) - 3));
4333 pCtx->eip += sizeof(s_abW2kSp4XmmZero) - 3;
4334 return true;
4335 }
4336 Log9(("pgmHandlePageZeroingCode: not page zeroing - 32-bit\n"));
4337 }
4338 else if (!pCtx->eflags.Bits.u1VM)
4339 Log9(("pgmHandlePageZeroingCode: not page zeroing - 16-bit\n"));
4340 else
4341 Log9(("pgmHandlePageZeroingCode: not page zeroing - v86\n"));
4342 }
4343 return false;
4344}
4345
4346#endif /* PGM_WITH_PAGE_ZEROING_DETECTION */
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette