PGMAllGst.h@ 105844

Last change on this file since 105844 was 104939, checked in by vboxsync, 6 months ago
VMM/PGM: Elimintate the GstWalk local from PGM_GST_NAME(WalkFast). bugref:10687
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 41.7 KB

Line
1	/* $Id: PGMAllGst.h 104939 2024-06-15 11:10:19Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Guest Paging Template - All context code.
4	*/
5
6	/*
7	* Copyright (C) 2006-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Internal Functions *
31	*********************************************************************************************************************************/
32	RT_C_DECLS_BEGIN
33	/** @todo Do we really need any of these forward declarations? */
34	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
35	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
36	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
37	DECLINLINE(int) PGM_GST_NAME(Walk)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PGSTPTWALK pGstWalk);
38	#endif
39	PGM_GST_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
40	PGM_GST_DECL(int, GetPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk);
41	PGM_GST_DECL(int, ModifyPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask);
42	PGM_GST_DECL(int, Exit)(PVMCPUCC pVCpu);
43
44	#ifdef IN_RING3 /* r3 only for now. */
45	PGM_GST_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta);
46	#endif
47	RT_C_DECLS_END
48
49
50	/**
51	* Enters the guest mode.
52	*
53	* @returns VBox status code.
54	* @param pVCpu The cross context virtual CPU structure.
55	* @param GCPhysCR3 The physical address from the CR3 register.
56	*/
57	PGM_GST_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
58	{
59	/*
60	* Map and monitor CR3
61	*/
62	uintptr_t idxBth = pVCpu->pgm.s.idxBothModeData;
63	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
64	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
65	return g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
66	}
67
68
69	/**
70	* Exits the guest mode.
71	*
72	* @returns VBox status code.
73	* @param pVCpu The cross context virtual CPU structure.
74	*/
75	PGM_GST_DECL(int, Exit)(PVMCPUCC pVCpu)
76	{
77	uintptr_t idxBth = pVCpu->pgm.s.idxBothModeData;
78	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
79	AssertReturn(g_aPgmBothModeData[idxBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
80	return g_aPgmBothModeData[idxBth].pfnUnmapCR3(pVCpu);
81	}
82
83
84	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
85	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
86	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
87
88
89	DECLINLINE(int) PGM_GST_NAME(WalkReturnNotPresent)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, uint8_t uLevel)
90	{
91	NOREF(pVCpu);
92	pWalk->fNotPresent = true;
93	pWalk->uLevel = uLevel;
94	pWalk->fFailed = PGM_WALKFAIL_NOT_PRESENT
95	\| ((uint32_t)uLevel << PGM_WALKFAIL_LEVEL_SHIFT);
96	return VERR_PAGE_TABLE_NOT_PRESENT;
97	}
98
99	DECLINLINE(int) PGM_GST_NAME(WalkReturnBadPhysAddr)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, uint8_t uLevel, int rc)
100	{
101	AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); NOREF(rc); NOREF(pVCpu);
102	pWalk->fBadPhysAddr = true;
103	pWalk->uLevel = uLevel;
104	pWalk->fFailed = PGM_WALKFAIL_BAD_PHYSICAL_ADDRESS
105	\| ((uint32_t)uLevel << PGM_WALKFAIL_LEVEL_SHIFT);
106	return VERR_PAGE_TABLE_NOT_PRESENT;
107	}
108
109	DECLINLINE(int) PGM_GST_NAME(WalkReturnRsvdError)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, uint8_t uLevel)
110	{
111	NOREF(pVCpu);
112	pWalk->fRsvdError = true;
113	pWalk->uLevel = uLevel;
114	pWalk->fFailed = PGM_WALKFAIL_RESERVED_BITS
115	\| ((uint32_t)uLevel << PGM_WALKFAIL_LEVEL_SHIFT);
116	return VERR_PAGE_TABLE_NOT_PRESENT;
117	}
118
119
120	/**
121	* Performs a guest page table walk.
122	*
123	* @returns VBox status code.
124	* @retval VINF_SUCCESS on success.
125	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
126	*
127	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
128	* @param GCPtr The guest virtual address to walk by.
129	* @param pWalk The page walk info.
130	* @param pGstWalk The guest mode specific page walk info.
131	* @thread EMT(pVCpu)
132	*/
133	DECLINLINE(int) PGM_GST_NAME(Walk)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PGSTPTWALK pGstWalk)
134	{
135	int rc;
136
137	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
138	/** @def PGM_GST_SLAT_WALK
139	* Macro to perform guest second-level address translation (EPT or Nested).
140	*
141	* @param a_pVCpu The cross context virtual CPU structure of the calling
142	* EMT.
143	* @param a_GCPtrNested The nested-guest linear address that caused the
144	* second-level translation.
145	* @param a_GCPhysNested The nested-guest physical address to translate.
146	* @param a_GCPhysOut Where to store the guest-physical address (result).
147	*/
148	# define PGM_GST_SLAT_WALK(a_pVCpu, a_GCPtrNested, a_GCPhysNested, a_GCPhysOut, a_pWalk) \
149	do { \
150	if ((a_pVCpu)->pgm.s.enmGuestSlatMode == PGMSLAT_EPT) \
151	{ \
152	PGMPTWALK WalkSlat; \
153	PGMPTWALKGST WalkGstSlat; \
154	int const rcX = pgmGstSlatWalk(a_pVCpu, a_GCPhysNested, true /* fIsLinearAddrValid */, a_GCPtrNested, &WalkSlat, \
155	&WalkGstSlat); \
156	if (RT_SUCCESS(rcX)) \
157	(a_GCPhysOut) = WalkSlat.GCPhys; \
158	else \
159	{ \
160	*(a_pWalk) = WalkSlat; \
161	return rcX; \
162	} \
163	} \
164	} while (0)
165	# endif
166
167	/*
168	* Init the walking structures.
169	*/
170	RT_ZERO(*pWalk);
171	RT_ZERO(*pGstWalk);
172	pWalk->GCPtr = GCPtr;
173
174	# if PGM_GST_TYPE == PGM_TYPE_32BIT \
175	\|\| PGM_GST_TYPE == PGM_TYPE_PAE
176	/*
177	* Boundary check for PAE and 32-bit (prevents trouble further down).
178	*/
179	if (RT_UNLIKELY(GCPtr >= _4G))
180	return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 8);
181	# endif
182
183	uint64_t fEffective;
184	{
185	# if PGM_GST_TYPE == PGM_TYPE_AMD64
186	/*
187	* The PML4 table.
188	*/
189	rc = pgmGstGetLongModePML4PtrEx(pVCpu, &pGstWalk->pPml4);
190	if (RT_SUCCESS(rc)) { /* probable */ }
191	else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 4, rc);
192
193	PX86PML4E pPml4e;
194	pGstWalk->pPml4e = pPml4e = &pGstWalk->pPml4->a[(GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK];
195	X86PML4E Pml4e;
196	pGstWalk->Pml4e.u = Pml4e.u = pPml4e->u;
197
198	if (GST_IS_PGENTRY_PRESENT(pVCpu, Pml4e)) { /* probable */ }
199	else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 4);
200
201	if (RT_LIKELY(GST_IS_PML4E_VALID(pVCpu, Pml4e))) { /* likely */ }
202	else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 4);
203
204	fEffective = Pml4e.u & ( X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_PWT \| X86_PML4E_PCD \| X86_PML4E_A
205	\| X86_PML4E_NX);
206	pWalk->fEffective = fEffective;
207
208	/*
209	* The PDPT.
210	*/
211	RTGCPHYS GCPhysPdpt = Pml4e.u & X86_PML4E_PG_MASK;
212	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
213	PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPdpt, GCPhysPdpt, pWalk);
214	# endif
215	rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhysPdpt, &pGstWalk->pPdpt);
216	if (RT_SUCCESS(rc)) { /* probable */ }
217	else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 3, rc);
218
219	# elif PGM_GST_TYPE == PGM_TYPE_PAE
220	rc = pgmGstGetPaePDPTPtrEx(pVCpu, &pGstWalk->pPdpt);
221	if (RT_SUCCESS(rc)) { /* probable */ }
222	else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 8, rc);
223	# endif
224	}
225	{
226	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
227	PX86PDPE pPdpe;
228	pGstWalk->pPdpe = pPdpe = &pGstWalk->pPdpt->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK];
229	X86PDPE Pdpe;
230	pGstWalk->Pdpe.u = Pdpe.u = pPdpe->u;
231
232	if (GST_IS_PGENTRY_PRESENT(pVCpu, Pdpe)) { /* probable */ }
233	else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 3);
234
235	if (RT_LIKELY(GST_IS_PDPE_VALID(pVCpu, Pdpe))) { /* likely */ }
236	else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 3);
237
238	# if PGM_GST_TYPE == PGM_TYPE_AMD64
239	fEffective &= (Pdpe.u & ( X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US
240	\| X86_PDPE_PWT \| X86_PDPE_PCD \| X86_PDPE_A));
241	fEffective \|= Pdpe.u & X86_PDPE_LM_NX;
242	# else
243	/*
244	* NX in the legacy-mode PAE PDPE is reserved. The valid check above ensures the NX bit is not set.
245	* The RW, US, A bits MBZ in PAE PDPTE entries but must be 1 the way we compute cumulative (effective) access rights.
246	*/
247	Assert(!(Pdpe.u & X86_PDPE_LM_NX));
248	fEffective = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A
249	\| (Pdpe.u & (X86_PDPE_PWT \| X86_PDPE_PCD));
250	# endif
251	pWalk->fEffective = fEffective;
252
253	/*
254	* The PD.
255	*/
256	RTGCPHYS GCPhysPd = Pdpe.u & X86_PDPE_PG_MASK;
257	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
258	PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPd, GCPhysPd, pWalk);
259	# endif
260	rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhysPd, &pGstWalk->pPd);
261	if (RT_SUCCESS(rc)) { /* probable */ }
262	else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 2, rc);
263
264	# elif PGM_GST_TYPE == PGM_TYPE_32BIT
265	rc = pgmGstGet32bitPDPtrEx(pVCpu, &pGstWalk->pPd);
266	if (RT_SUCCESS(rc)) { /* probable */ }
267	else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 8, rc);
268	# endif
269	}
270	{
271	PGSTPDE pPde;
272	pGstWalk->pPde = pPde = &pGstWalk->pPd->a[(GCPtr >> GST_PD_SHIFT) & GST_PD_MASK];
273	GSTPDE Pde;
274	pGstWalk->Pde.u = Pde.u = pPde->u;
275	if (GST_IS_PGENTRY_PRESENT(pVCpu, Pde)) { /* probable */ }
276	else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 2);
277	if ((Pde.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu))
278	{
279	if (RT_LIKELY(GST_IS_BIG_PDE_VALID(pVCpu, Pde))) { /* likely */ }
280	else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 2);
281
282	/*
283	* We're done.
284	*/
285	# if PGM_GST_TYPE == PGM_TYPE_32BIT
286	fEffective = Pde.u & (X86_PDE4M_P \| X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PWT \| X86_PDE4M_PCD \| X86_PDE4M_A);
287	# else
288	fEffective &= Pde.u & (X86_PDE4M_P \| X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PWT \| X86_PDE4M_PCD \| X86_PDE4M_A);
289	fEffective \|= Pde.u & X86_PDE2M_PAE_NX;
290	# endif
291	fEffective \|= Pde.u & (X86_PDE4M_D \| X86_PDE4M_G);
292	fEffective \|= (Pde.u & X86_PDE4M_PAT) >> X86_PDE4M_PAT_SHIFT;
293	pWalk->fEffective = fEffective;
294	Assert(GST_IS_NX_ACTIVE(pVCpu) \|\| !(fEffective & PGM_PTATTRS_NX_MASK));
295	Assert(fEffective & PGM_PTATTRS_R_MASK);
296
297	pWalk->fBigPage = true;
298	pWalk->fSucceeded = true;
299	RTGCPHYS GCPhysPde = GST_GET_BIG_PDE_GCPHYS(pVCpu->CTX_SUFF(pVM), Pde)
300	\| (GCPtr & GST_BIG_PAGE_OFFSET_MASK);
301	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
302	PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPde, GCPhysPde, pWalk);
303	# endif
304	pWalk->GCPhys = GCPhysPde;
305	PGM_A20_APPLY_TO_VAR(pVCpu, pWalk->GCPhys); /** @todo why do we apply it here and not below?!? */
306	return VINF_SUCCESS;
307	}
308
309	if (RT_UNLIKELY(!GST_IS_PDE_VALID(pVCpu, Pde)))
310	return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 2);
311	# if PGM_GST_TYPE == PGM_TYPE_32BIT
312	fEffective = Pde.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD \| X86_PDE_A);
313	# else
314	fEffective &= Pde.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD \| X86_PDE_A);
315	fEffective \|= Pde.u & X86_PDE_PAE_NX;
316	# endif
317	pWalk->fEffective = fEffective;
318
319	/*
320	* The PT.
321	*/
322	RTGCPHYS GCPhysPt = GST_GET_PDE_GCPHYS(Pde);
323	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
324	PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPt, GCPhysPt, pWalk);
325	# endif
326	rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhysPt, &pGstWalk->pPt);
327	if (RT_SUCCESS(rc)) { /* probable */ }
328	else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 1, rc);
329	}
330	{
331	PGSTPTE pPte;
332	pGstWalk->pPte = pPte = &pGstWalk->pPt->a[(GCPtr >> GST_PT_SHIFT) & GST_PT_MASK];
333	GSTPTE Pte;
334	pGstWalk->Pte.u = Pte.u = pPte->u;
335
336	if (GST_IS_PGENTRY_PRESENT(pVCpu, Pte)) { /* probable */ }
337	else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 1);
338
339	if (RT_LIKELY(GST_IS_PTE_VALID(pVCpu, Pte))) { /* likely */ }
340	else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 1);
341
342	/*
343	* We're done.
344	*/
345	fEffective &= Pte.u & (X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_A);
346	fEffective \|= Pte.u & (X86_PTE_D \| X86_PTE_PAT \| X86_PTE_G);
347	# if PGM_GST_TYPE != PGM_TYPE_32BIT
348	fEffective \|= Pte.u & X86_PTE_PAE_NX;
349	# endif
350	pWalk->fEffective = fEffective;
351	Assert(GST_IS_NX_ACTIVE(pVCpu) \|\| !(fEffective & PGM_PTATTRS_NX_MASK));
352	Assert(fEffective & PGM_PTATTRS_R_MASK);
353
354	pWalk->fSucceeded = true;
355	RTGCPHYS GCPhysPte = GST_GET_PTE_GCPHYS(Pte)
356	\| (GCPtr & GUEST_PAGE_OFFSET_MASK);
357	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
358	PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPte, GCPhysPte, pWalk);
359	# endif
360	pWalk->GCPhys = GCPhysPte;
361	return VINF_SUCCESS;
362	}
363	}
364
365	#endif /* 32BIT, PAE, AMD64 */
366
367	/**
368	* Gets effective Guest OS page information.
369	*
370	* @returns VBox status code.
371	* @param pVCpu The cross context virtual CPU structure.
372	* @param GCPtr Guest Context virtual address of the page.
373	* @param pWalk Where to store the page walk info.
374	* @thread EMT(pVCpu)
375	*/
376	PGM_GST_DECL(int, GetPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk)
377	{
378	#if PGM_GST_TYPE == PGM_TYPE_REAL \
379	\|\| PGM_GST_TYPE == PGM_TYPE_PROT
380
381	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
382	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
383	{
384	PGMPTWALK WalkSlat;
385	PGMPTWALKGST WalkGstSlat;
386	int const rc = pgmGstSlatWalk(pVCpu, GCPtr, true /* fIsLinearAddrValid */, GCPtr, &WalkSlat, &WalkGstSlat);
387	if (RT_SUCCESS(rc))
388	{
389	RT_ZERO(*pWalk);
390	pWalk->fSucceeded = true;
391	pWalk->GCPtr = GCPtr;
392	pWalk->GCPhys = WalkSlat.GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
393	pWalk->fEffective = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US;
394	}
395	else
396	*pWalk = WalkSlat;
397	return rc;
398	}
399	# endif
400
401	/*
402	* Fake it.
403	*/
404	RT_ZERO(*pWalk);
405	pWalk->fSucceeded = true;
406	pWalk->GCPtr = GCPtr;
407	pWalk->GCPhys = GCPtr & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
408	pWalk->fEffective = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US;
409	NOREF(pVCpu);
410	return VINF_SUCCESS;
411
412	#elif PGM_GST_TYPE == PGM_TYPE_32BIT \
413	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
414	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
415
416	GSTPTWALK GstWalk;
417	int rc = PGM_GST_NAME(Walk)(pVCpu, GCPtr, pWalk, &GstWalk);
418	if (RT_FAILURE(rc))
419	return rc;
420
421	Assert(pWalk->fSucceeded);
422	Assert(pWalk->GCPtr == GCPtr);
423
424	PGMPTATTRS fFlags;
425	if (!pWalk->fBigPage)
426	fFlags = (GstWalk.Pte.u & ~(GST_PTE_PG_MASK \| X86_PTE_RW \| X86_PTE_US)) /* NX not needed */
427	\| (pWalk->fEffective & (PGM_PTATTRS_W_MASK \| PGM_PTATTRS_US_MASK))
428	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_GST_TYPE)
429	\| (pWalk->fEffective & PGM_PTATTRS_NX_MASK)
430	# endif
431	;
432	else
433	fFlags = (GstWalk.Pde.u & ~(GST_PTE_PG_MASK \| X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PS)) /* NX not needed */
434	\| (pWalk->fEffective & (PGM_PTATTRS_W_MASK \| PGM_PTATTRS_US_MASK \| PGM_PTATTRS_PAT_MASK))
435	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_GST_TYPE)
436	\| (pWalk->fEffective & PGM_PTATTRS_NX_MASK)
437	# endif
438	;
439
440	pWalk->GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
441	pWalk->fEffective = fFlags;
442	return VINF_SUCCESS;
443
444	#else
445	# error "shouldn't be here!"
446	/* something else... */
447	return VERR_NOT_SUPPORTED;
448	#endif
449	}
450
451
452	/* x x x x x x x x */
453	/* x x x x x x x x */
454
455	#if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) \|\| defined(VBOX_WITH_NESTED_HWVIRT_SVM_XXX) \|\| defined(DOXYGEN_RUNNING)
456	/** Converts regular style walk info to fast style. */
457	DECL_FORCE_INLINE(void) PGM_GST_NAME(ConvertPtWalkToFast)(PGMPTWALK const *pSrc, PPGMPTWALKFAST pDst)
458	{
459	pDst->GCPtr = pSrc->GCPtr;
460	pDst->GCPhys = pSrc->GCPhys;
461	pDst->GCPhysNested = pSrc->GCPhysNested;
462	pDst->fInfo = (pSrc->fSucceeded ? PGM_WALKINFO_SUCCEEDED : 0)
463	\| (pSrc->fIsSlat ? PGM_WALKINFO_IS_SLAT : 0)
464	\| (pSrc->fIsLinearAddrValid ? PGM_WALKINFO_IS_LINEAR_ADDR_VALID : 0);
465	pDst->fFailed = pSrc->fFailed \| ((uint32_t)pSrc->uLevel << PGM_WALKFAIL_LEVEL_SHIFT);
466	pDst->fEffective = pSrc->fEffective;
467	}
468	#endif
469
470
471	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
472	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
473	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
474
475	DECLINLINE(int) PGM_GST_NAME(WalkFastReturnNotPresent)(PVMCPUCC pVCpu, PPGMPTWALKFAST pWalk, uint8_t uLevel)
476	{
477	RT_NOREF(pVCpu);
478	pWalk->fFailed = PGM_WALKFAIL_NOT_PRESENT \| ((uint32_t)uLevel << PGM_WALKFAIL_LEVEL_SHIFT);
479	return VERR_PAGE_TABLE_NOT_PRESENT;
480	}
481
482	DECLINLINE(int) PGM_GST_NAME(WalkFastReturnBadPhysAddr)(PVMCPUCC pVCpu, PPGMPTWALKFAST pWalk, uint8_t uLevel, int rc)
483	{
484	AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); RT_NOREF(pVCpu, rc);
485	pWalk->fFailed = PGM_WALKFAIL_BAD_PHYSICAL_ADDRESS \| ((uint32_t)uLevel << PGM_WALKFAIL_LEVEL_SHIFT);
486	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
487	}
488
489	DECLINLINE(int) PGM_GST_NAME(WalkFastReturnRsvdError)(PVMCPUCC pVCpu, PPGMPTWALKFAST pWalk, uint8_t uLevel)
490	{
491	RT_NOREF(pVCpu);
492	pWalk->fFailed = PGM_WALKFAIL_RESERVED_BITS \| ((uint32_t)uLevel << PGM_WALKFAIL_LEVEL_SHIFT);
493	return VERR_RESERVED_PAGE_TABLE_BITS;
494	}
495
496	# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) \|\| defined(VBOX_WITH_NESTED_HWVIRT_SVM_XXX) \|\| defined(DOXYGEN_RUNNING)
497	/** @def PGM_GST_SLAT_WALK_FAST
498	* Macro to perform guest second-level address translation (EPT or Nested).
499	*
500	* @param a_pVCpu The cross context virtual CPU structure of the calling
501	* EMT.
502	* @param a_GCPtrNested The nested-guest linear address that caused the
503	* second-level translation.
504	* @param a_GCPhysNested The nested-guest physical address to translate.
505	* @param a_fFinal Set to @a true if this is the final page table entry
506	* and effective nested page table flags should be
507	* merged into PGMPTWALKFAST::fEffective. Otherwise
508	* set to @a false and nothing done.
509	* @param a_GCPhysOut Where to store the guest-physical address (result).
510	* @param a_pWalk The @a pWalk argument to the function.
511	*/
512	# define PGM_GST_SLAT_WALK_FAST(a_pVCpu, a_GCPtrNested, a_GCPhysNested, a_fFinal, a_GCPhysOut, a_pWalk) \
513	do { \
514	/** @todo Optimize this. Among other things, WalkSlat can be eliminated. WalkGstSlat is completely pointless. */ \
515	/** @todo pass fFlags along as appropriate... */ \
516	if (a_enmGuestSlatMode != PGMSLAT_DIRECT) \
517	{ \
518	PGMPTWALK WalkSlat; \
519	PGMPTWALKGST WalkGstSlat; \
520	int rcX; \
521	if (a_enmGuestSlatMode == PGMSLAT_EPT) \
522	rcX = PGM_GST_SLAT_NAME_EPT(Walk)(a_pVCpu, a_GCPhysNested, true /* fIsLinearAddrValid */, a_GCPtrNested, \
523	&WalkSlat, &WalkGstSlat.u.Ept); \
524	else AssertFailedReturn(VERR_NOT_IMPLEMENTED); \
525	if (RT_SUCCESS(rcX)) \
526	(a_GCPhysOut) = WalkSlat.GCPhys; \
527	else \
528	{ \
529	PGM_GST_NAME(ConvertPtWalkToFast)(&WalkSlat, pWalk); \
530	return rcX; \
531	} \
532	if (a_fFinal) \
533	{ /* Merge in the nested paging flags for the final GCPhys. */ \
534	if (a_enmGuestSlatMode == PGMSLAT_EPT) \
535	(a_pWalk)->fEffective = ((a_pWalk)->fEffective & ~PGM_PTATTRS_EPT_MASK) \
536	\| (WalkSlat.fEffective & PGM_PTATTRS_EPT_MASK); \
537	else AssertFailedReturn(VERR_NOT_IMPLEMENTED); \
538	} \
539	} \
540	} while (0)
541	# else
542	# define PGM_GST_SLAT_WALK_FAST(a_pVCpu, a_GCPtrNested, a_GCPhysNested, a_fFinal, a_GCPhysOut, a_pWalk) do { } while (0)
543	# endif
544	# if PGM_GST_TYPE == PGM_TYPE_32BIT
545	# define PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, a_fEffective, a_pEntryU, a_OrgEntryU, a_fFlags) do { \
546	if (!a_fSetFlags \|\| ((a_OrgEntryU) & (a_fFlags)) == (a_fFlags)) \
547	{ /* likely */ } \
548	else \
549	{ \
550	ASMAtomicOrU32((a_pEntryU), (a_fFlags)); \
551	(a_fEffective) \|= (a_fFlags); \
552	} \
553	} while (0)
554	# else
555	# define PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, a_fEffective, a_pEntryU, a_OrgEntryU, a_fFlags) do { \
556	if (!a_fSetFlags \|\| ((a_OrgEntryU) & (a_fFlags)) == (a_fFlags)) \
557	{ /* likely */ } \
558	else \
559	{ \
560	ASMAtomicOrU64((a_pEntryU), (a_fFlags)); \
561	(a_fEffective) \|= (a_fFlags); \
562	} \
563	} while (0)
564	# endif
565
566	/**
567	* Performs a guest page table walk.
568	*
569	* @returns VBox status code.
570	* @retval VINF_SUCCESS on success.
571	* @retval VERR_PAGE_TABLE_NOT_PRESENT, VERR_RESERVED_PAGE_TABLE_BITS or
572	* VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS on normal failure.
573	* The failure reason is also recorded in PGMPTWALKFAST::fFailed.
574	*
575	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
576	* @param GCPtr The guest virtual address to walk by.
577	* @param fFlags PGMQPAGE_F_XXX.
578	* This is ignored when @a a_fSetFlags is @c false.
579	* @param pWalk The page walk info.
580	* @tparam a_enmGuestSlatMode The SLAT mode of the function.
581	* @tparam a_fSetFlags Whether to process @a fFlags and set accessed
582	* and dirty flags accordingly.
583	* @thread EMT(pVCpu)
584	*/
585	template<PGMSLAT const a_enmGuestSlatMode = PGMSLAT_DIRECT, bool const a_fSetFlags = false>
586	DECLINLINE(int) PGM_GST_NAME(WalkFast)(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fFlags, PPGMPTWALKFAST pWalk)
587	{
588	int rc;
589
590	/*
591	* Init the walking structures.
592	*/
593	pWalk->GCPtr = GCPtr;
594	pWalk->GCPhys = 0;
595	pWalk->GCPhysNested = 0;
596	pWalk->fInfo = 0;
597	pWalk->fFailed = 0;
598	pWalk->fEffective = 0;
599
600	# if PGM_GST_TYPE == PGM_TYPE_32BIT \
601	\|\| PGM_GST_TYPE == PGM_TYPE_PAE
602	/*
603	* Boundary check for PAE and 32-bit (prevents trouble further down).
604	*/
605	if (RT_LIKELY(GCPtr < _4G))
606	{ /* extremely likely */ }
607	else
608	return PGM_GST_NAME(WalkFastReturnNotPresent)(pVCpu, pWalk, 8);
609	# endif
610
611	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
612	PX86PDPT pPdpt;
613	# endif
614	uint64_t fEffective;
615	{
616	# if PGM_GST_TYPE == PGM_TYPE_AMD64
617	/*
618	* The PML4 table.
619	*/
620	PX86PML4 pPml4;
621	rc = pgmGstGetLongModePML4PtrEx(pVCpu, &pPml4);
622	if (RT_SUCCESS(rc)) { /* probable */ }
623	else return PGM_GST_NAME(WalkFastReturnBadPhysAddr)(pVCpu, pWalk, 4, rc);
624
625	PX86PML4E pPml4e;
626	pPml4e = &pPml4->a[(GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK];
627	X86PML4E Pml4e;
628	Pml4e.u = ASMAtomicUoReadU64(&pPml4e->u);
629
630	if (GST_IS_PGENTRY_PRESENT(pVCpu, Pml4e)) { /* probable */ }
631	else return PGM_GST_NAME(WalkFastReturnNotPresent)(pVCpu, pWalk, 4);
632
633	if (RT_LIKELY(GST_IS_PML4E_VALID(pVCpu, Pml4e))) { /* likely */ }
634	else return PGM_GST_NAME(WalkFastReturnRsvdError)(pVCpu, pWalk, 4);
635
636	fEffective = Pml4e.u & ( X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_PWT \| X86_PML4E_PCD \| X86_PML4E_A
637	\| X86_PML4E_NX);
638	PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, fEffective, &pPml4e->u, Pml4e.u, X86_PML4E_A);
639	pWalk->fEffective = fEffective;
640
641	/*
642	* The PDPT.
643	*/
644	RTGCPHYS GCPhysPdpt = Pml4e.u & X86_PML4E_PG_MASK;
645	PGM_GST_SLAT_WALK_FAST(pVCpu, GCPtr, GCPhysPdpt, false /a_fFinal/, GCPhysPdpt, pWalk);
646	rc = pgmPhysGCPhys2CCPtrLockless(pVCpu, GCPhysPdpt, (void **)&pPdpt);
647	if (RT_SUCCESS(rc)) { /* probable */ }
648	else return PGM_GST_NAME(WalkFastReturnBadPhysAddr)(pVCpu, pWalk, 3, rc);
649
650	# elif PGM_GST_TYPE == PGM_TYPE_PAE
651	rc = pgmGstGetPaePDPTPtrEx(pVCpu, &pPdpt);
652	if (RT_SUCCESS(rc)) { /* probable */ }
653	else return PGM_GST_NAME(WalkFastReturnBadPhysAddr)(pVCpu, pWalk, 8, rc);
654	# endif
655	}
656	PGSTPD pPd;
657	{
658	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
659	PX86PDPE pPdpe;
660	pPdpe = &pPdpt->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK];
661	X86PDPE Pdpe;
662	Pdpe.u = ASMAtomicUoReadU64(&pPdpe->u);
663
664	if (GST_IS_PGENTRY_PRESENT(pVCpu, Pdpe)) { /* probable */ }
665	else return PGM_GST_NAME(WalkFastReturnNotPresent)(pVCpu, pWalk, 3);
666
667	if (RT_LIKELY(GST_IS_PDPE_VALID(pVCpu, Pdpe))) { /* likely */ }
668	else return PGM_GST_NAME(WalkFastReturnRsvdError)(pVCpu, pWalk, 3);
669
670	# if PGM_GST_TYPE == PGM_TYPE_AMD64
671	fEffective &= (Pdpe.u & ( X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US
672	\| X86_PDPE_PWT \| X86_PDPE_PCD \| X86_PDPE_A));
673	fEffective \|= Pdpe.u & X86_PDPE_LM_NX;
674	PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, fEffective, &pPdpe->u, Pdpe.u, X86_PDE_A);
675	# else
676	/*
677	* NX in the legacy-mode PAE PDPE is reserved. The valid check above ensures the NX bit is not set.
678	* The RW, US, A bits MBZ in PAE PDPTE entries but must be 1 the way we compute cumulative (effective) access rights.
679	*/
680	Assert(!(Pdpe.u & X86_PDPE_LM_NX));
681	fEffective = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A
682	\| (Pdpe.u & (X86_PDPE_PWT \| X86_PDPE_PCD));
683	# endif
684	pWalk->fEffective = fEffective;
685
686	/*
687	* The PD.
688	*/
689	RTGCPHYS GCPhysPd = Pdpe.u & X86_PDPE_PG_MASK;
690	PGM_GST_SLAT_WALK_FAST(pVCpu, GCPtr, GCPhysPd, false /a_fFinal/, GCPhysPd, pWalk);
691	rc = pgmPhysGCPhys2CCPtrLockless(pVCpu, GCPhysPd, (void **)&pPd);
692	if (RT_SUCCESS(rc)) { /* probable */ }
693	else return PGM_GST_NAME(WalkFastReturnBadPhysAddr)(pVCpu, pWalk, 2, rc);
694
695	# elif PGM_GST_TYPE == PGM_TYPE_32BIT
696	rc = pgmGstGet32bitPDPtrEx(pVCpu, &pPd);
697	if (RT_SUCCESS(rc)) { /* probable */ }
698	else return PGM_GST_NAME(WalkFastReturnBadPhysAddr)(pVCpu, pWalk, 8, rc);
699	# endif
700	}
701	PGSTPT pPt;
702	{
703	PGSTPDE pPde;
704	pPde = &pPd->a[(GCPtr >> GST_PD_SHIFT) & GST_PD_MASK];
705	GSTPDE Pde;
706	# if PGM_GST_TYPE != PGM_TYPE_32BIT
707	Pde.u = ASMAtomicUoReadU64(&pPde->u);
708	# else
709	Pde.u = ASMAtomicUoReadU32(&pPde->u);
710	# endif
711	if (GST_IS_PGENTRY_PRESENT(pVCpu, Pde)) { /* probable */ }
712	else return PGM_GST_NAME(WalkFastReturnNotPresent)(pVCpu, pWalk, 2);
713	if ((Pde.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu))
714	{
715	if (RT_LIKELY(GST_IS_BIG_PDE_VALID(pVCpu, Pde))) { /* likely */ }
716	else return PGM_GST_NAME(WalkFastReturnRsvdError)(pVCpu, pWalk, 2);
717
718	/*
719	* We're done.
720	*/
721	pWalk->fInfo = PGM_WALKINFO_SUCCEEDED \| PGM_WALKINFO_BIG_PAGE;
722
723	# if PGM_GST_TYPE == PGM_TYPE_32BIT
724	fEffective = Pde.u & (X86_PDE4M_P \| X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PWT \| X86_PDE4M_PCD \| X86_PDE4M_A);
725	# else
726	fEffective &= Pde.u & (X86_PDE4M_P \| X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PWT \| X86_PDE4M_PCD \| X86_PDE4M_A);
727	fEffective \|= Pde.u & X86_PDE2M_PAE_NX;
728	# endif
729	fEffective \|= Pde.u & (X86_PDE4M_D \| X86_PDE4M_G);
730	fEffective \|= (Pde.u & X86_PDE4M_PAT) >> X86_PDE4M_PAT_SHIFT;
731
732	rc = VINF_SUCCESS;
733	if (a_fSetFlags)
734	{
735	/* We have to validate the access before setting any flags. */
736	uint32_t fFailed = 0;
737	if ((fFlags & PGMQPAGE_F_USER_MODE) && !(fEffective & X86_PDE4M_US))
738	fFailed \|= PGM_WALKFAIL_NOT_ACCESSIBLE_BY_MODE;
739	if (fFlags & PGMQPAGE_F_WRITE)
740	{
741	if ( (fEffective & X86_PDE4M_RW)
742	\|\| (fFlags & (PGMQPAGE_F_USER_MODE \| PGMQPAGE_F_CR0_WP0)) == PGMQPAGE_F_CR0_WP0)
743	{ /* likely*/ }
744	else fFailed \|= PGM_WALKFAIL_NOT_WRITABLE;
745	}
746	# if PGM_GST_TYPE != PGM_TYPE_32BIT
747	else if (fFlags & PGMQPAGE_F_EXECUTE)
748	{
749	if (!(fEffective & X86_PDE2M_PAE_NX) \|\| !pVCpu->pgm.s.fNoExecuteEnabled) { /* likely */ }
750	else fFailed \|= PGM_WALKFAIL_NOT_EXECUTABLE;
751	}
752	# endif
753	if (fFailed == 0)
754	{
755	if (!(fFlags & PGMQPAGE_F_WRITE))
756	PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, fEffective, &pPde->u, Pde.u, X86_PDE4M_A);
757	else
758	PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, fEffective, &pPde->u, Pde.u, X86_PDE4M_A \| X86_PDE4M_D);
759	}
760	else
761	{
762	pWalk->fFailed = fFailed \| (2U << PGM_WALKFAIL_LEVEL_SHIFT);
763	pWalk->fInfo = PGM_WALKINFO_BIG_PAGE;
764	rc = VERR_ACCESS_DENIED;
765	}
766	}
767
768	pWalk->fEffective = fEffective;
769	Assert(GST_IS_NX_ACTIVE(pVCpu) \|\| !(fEffective & PGM_PTATTRS_NX_MASK));
770	Assert(fEffective & PGM_PTATTRS_R_MASK);
771
772	RTGCPHYS GCPhysPde = GST_GET_BIG_PDE_GCPHYS(pVCpu->CTX_SUFF(pVM), Pde)
773	\| (GCPtr & GST_BIG_PAGE_OFFSET_MASK);
774	PGM_GST_SLAT_WALK_FAST(pVCpu, GCPtr, GCPhysPde, true /a_fFinal/, GCPhysPde, pWalk);
775	pWalk->GCPhys = GCPhysPde;
776	PGM_A20_APPLY_TO_VAR(pVCpu, pWalk->GCPhys); /** @todo why do we apply it here and not below?!? */
777	return rc;
778	}
779
780	if (RT_UNLIKELY(!GST_IS_PDE_VALID(pVCpu, Pde)))
781	return PGM_GST_NAME(WalkFastReturnRsvdError)(pVCpu, pWalk, 2);
782	# if PGM_GST_TYPE == PGM_TYPE_32BIT
783	fEffective = Pde.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD \| X86_PDE_A);
784	# else
785	fEffective &= Pde.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD \| X86_PDE_A);
786	fEffective \|= Pde.u & X86_PDE_PAE_NX;
787	# endif
788	PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, fEffective, &pPde->u, Pde.u, X86_PDE_A);
789	pWalk->fEffective = fEffective;
790
791	/*
792	* The PT.
793	*/
794	RTGCPHYS GCPhysPt = GST_GET_PDE_GCPHYS(Pde);
795	PGM_GST_SLAT_WALK_FAST(pVCpu, GCPtr, GCPhysPt, false /a_fFinal/, GCPhysPt, pWalk);
796	rc = pgmPhysGCPhys2CCPtrLockless(pVCpu, GCPhysPt, (void **)&pPt);
797	if (RT_SUCCESS(rc)) { /* probable */ }
798	else return PGM_GST_NAME(WalkFastReturnBadPhysAddr)(pVCpu, pWalk, 1, rc);
799	}
800	{
801	PGSTPTE pPte;
802	pPte = &pPt->a[(GCPtr >> GST_PT_SHIFT) & GST_PT_MASK];
803	GSTPTE Pte;
804	# if PGM_GST_TYPE != PGM_TYPE_32BIT
805	Pte.u = ASMAtomicUoReadU64(&pPte->u);
806	# else
807	Pte.u = ASMAtomicUoReadU32(&pPte->u);
808	# endif
809
810	if (GST_IS_PGENTRY_PRESENT(pVCpu, Pte)) { /* probable */ }
811	else return PGM_GST_NAME(WalkFastReturnNotPresent)(pVCpu, pWalk, 1);
812
813	if (RT_LIKELY(GST_IS_PTE_VALID(pVCpu, Pte))) { /* likely */ }
814	else return PGM_GST_NAME(WalkFastReturnRsvdError)(pVCpu, pWalk, 1);
815
816	/*
817	* We're done.
818	*/
819	pWalk->fInfo = PGM_WALKINFO_SUCCEEDED;
820
821	fEffective &= Pte.u & (X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_A);
822	# if PGM_GST_TYPE != PGM_TYPE_32BIT
823	fEffective \|= Pte.u & (X86_PTE_D \| X86_PTE_PAT \| X86_PTE_G \| X86_PTE_PAE_NX);
824	# else
825	fEffective \|= Pte.u & (X86_PTE_D \| X86_PTE_PAT \| X86_PTE_G);
826	# endif
827
828	rc = VINF_SUCCESS;
829	if (a_fSetFlags)
830	{
831	/* We have to validate the access before setting any flags. */
832	uint32_t fFailed = 0;
833	if ((fFlags & PGMQPAGE_F_USER_MODE) && !(fEffective & X86_PTE_US))
834	fFailed \|= PGM_WALKFAIL_NOT_ACCESSIBLE_BY_MODE;
835	if (fFlags & PGMQPAGE_F_WRITE)
836	{
837	if ((fEffective & X86_PTE_RW) \|\| (fFlags & (PGMQPAGE_F_USER_MODE \| PGMQPAGE_F_CR0_WP0)) == PGMQPAGE_F_CR0_WP0)
838	{ /* likely*/ }
839	else fFailed \|= PGM_WALKFAIL_NOT_WRITABLE;
840	}
841	# if PGM_GST_TYPE != PGM_TYPE_32BIT
842	else if (fFlags & PGMQPAGE_F_EXECUTE)
843	{
844	if (!(fEffective & X86_PTE_PAE_NX) \|\| !pVCpu->pgm.s.fNoExecuteEnabled) { /* likely */ }
845	else fFailed \|= PGM_WALKFAIL_NOT_EXECUTABLE;
846	}
847	# endif
848	if (fFailed == 0)
849	{
850	if (!(fFlags & PGMQPAGE_F_WRITE))
851	PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, fEffective, &pPte->u, Pte.u, X86_PTE_A);
852	else
853	PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, fEffective, &pPte->u, Pte.u, X86_PTE_A \| X86_PTE_D);
854	}
855	else
856	{
857	pWalk->fFailed = fFailed \| (1U << PGM_WALKFAIL_LEVEL_SHIFT);
858	pWalk->fInfo = 0;
859	rc = VERR_ACCESS_DENIED;
860	}
861	}
862
863	pWalk->fEffective = fEffective;
864	Assert(GST_IS_NX_ACTIVE(pVCpu) \|\| !(fEffective & PGM_PTATTRS_NX_MASK));
865	Assert(fEffective & PGM_PTATTRS_R_MASK);
866
867	RTGCPHYS GCPhysPte = GST_GET_PTE_GCPHYS(Pte)
868	\| (GCPtr & GUEST_PAGE_OFFSET_MASK);
869	PGM_GST_SLAT_WALK_FAST(pVCpu, GCPtr, GCPhysPte, true /a_fFinal/, GCPhysPte, pWalk);
870	pWalk->GCPhys = GCPhysPte;
871	return rc;
872	}
873	}
874
875	# undef PGM_GST_SLAT_WALK_FAST
876	# undef PGM_GST_ENSURE_ENTRY_FLAGS_SET
877
878	#endif /* 32BIT, PAE, AMD64 */
879
880	/**
881	* Guest virtual to guest physical + info translation, the faster and better
882	* version.
883	*
884	* @returns VBox status code.
885	* @param pVCpu The cross context virtual CPU structure.
886	* @param GCPtr Guest Context virtual address of the page.
887	* @param fFlags PGMQPAGE_F_XXX
888	* @param pWalk Where to store the page walk info.
889	* @thread EMT(pVCpu)
890	*/
891	#define PGM_GET_PAGE_F_WRITE
892	#define PGM_GET_PAGE_F_READ
893	PGM_GST_DECL(int, QueryPageFast)(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fFlags, PPGMPTWALKFAST pWalk)
894	{
895	#if PGM_GST_TYPE == PGM_TYPE_REAL \
896	\|\| PGM_GST_TYPE == PGM_TYPE_PROT
897
898	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
899	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
900	{
901	/** @todo optimize this case as well. */
902	/** @todo pass fFlags along. */
903	PGMPTWALK WalkSlat;
904	PGMPTWALKGST WalkGstSlat;
905	int const rc = pgmGstSlatWalk(pVCpu, GCPtr, true /* fIsLinearAddrValid */, GCPtr, &WalkSlat, &WalkGstSlat);
906	if (RT_SUCCESS(rc))
907	{
908	PGMPTWALKFAST_ZERO(pWalk);
909	pWalk->GCPtr = GCPtr;
910	pWalk->GCPhys = WalkSlat.GCPhys;
911	pWalk->GCPhysNested = 0;
912	pWalk->fInfo = PGM_WALKINFO_SUCCEEDED;
913	pWalk->fFailed = PGM_WALKFAIL_SUCCESS;
914	pWalk->fEffective = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D;
915	}
916	else
917	PGM_GST_NAME(ConvertPtWalkToFast)(&WalkSlat, pWalk);
918	return rc;
919	}
920	# endif
921
922	/*
923	* Fake it.
924	*/
925	pWalk->GCPtr = GCPtr;
926	pWalk->GCPhys = GCPtr;
927	pWalk->GCPhysNested = 0;
928	pWalk->fInfo = PGM_WALKINFO_SUCCEEDED;
929	pWalk->fFailed = PGM_WALKFAIL_SUCCESS;
930	pWalk->fEffective = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D;
931	RT_NOREF(pVCpu, fFlags);
932	return VINF_SUCCESS;
933
934	#elif PGM_GST_TYPE == PGM_TYPE_32BIT \
935	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
936	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
937
938	int rc;
939	# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) \|\| defined(VBOX_WITH_NESTED_HWVIRT_SVM_XXX)
940	switch (pVCpu->pgm.s.enmGuestSlatMode)
941	{
942	case PGMSLAT_DIRECT:
943	# endif
944	if (fFlags)
945	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_DIRECT, true>(pVCpu, GCPtr, fFlags, pWalk);
946	else
947	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_DIRECT, false>(pVCpu, GCPtr, 0, pWalk);
948	# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) \|\| defined(VBOX_WITH_NESTED_HWVIRT_SVM_XXX)
949	break;
950	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
951	case PGMSLAT_EPT:
952	if (fFlags)
953	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_EPT, true>(pVCpu, GCPtr, fFlags, pWalk);
954	else
955	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_EPT, false>(pVCpu, GCPtr, 0, pWalk);
956	break;
957	# endif
958	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM_XXX
959	case PGMSLAT_32BIT:
960	if (fFlags)
961	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_32BIT, true>(pVCpu, GCPtr, fFlags, pWalk);
962	else
963	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_32BIT, false>(pVCpu, GCPtr, 0, pWalk);
964	break;
965	case PGMSLAT_PAE:
966	if (fFlags)
967	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_PAE, true>(pVCpu, GCPtr, fFlags, pWalk);
968	else
969	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_PAE, false>(pVCpu, GCPtr, 0, pWalk);
970	break;
971	case PGMSLAT_AMD64:
972	if (fFlags)
973	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_AMD64, true>(pVCpu, GCPtr, fFlags, pWalk);
974	else
975	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_AMD64, false>(pVCpu, GCPtr, 0, pWalk);
976	break;
977	# endif
978	default:
979	AssertFailedReturn(VERR_INTERNAL_ERROR_4);
980	}
981	# endif
982	if (RT_SUCCESS(rc))
983	{
984	Assert(pWalk->fInfo & PGM_WALKINFO_SUCCEEDED);
985	Assert(pWalk->GCPtr == GCPtr);
986	Assert((pWalk->GCPhys & GUEST_PAGE_OFFSET_MASK) == (GCPtr & GUEST_PAGE_OFFSET_MASK));
987	return VINF_SUCCESS;
988	}
989	return rc;
990
991	#else
992	# error "shouldn't be here!"
993	/* something else... */
994	return VERR_NOT_SUPPORTED;
995	#endif
996	}
997
998	/* x x x x x x x x */
999	/* x x x x x x x x */
1000
1001	/**
1002	* Modify page flags for a range of pages in the guest's tables
1003	*
1004	* The existing flags are ANDed with the fMask and ORed with the fFlags.
1005	*
1006	* @returns VBox status code.
1007	* @param pVCpu The cross context virtual CPU structure.
1008	* @param GCPtr Virtual address of the first page in the range. Page aligned!
1009	* @param cb Size (in bytes) of the page range to apply the modification to. Page aligned!
1010	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
1011	* @param fMask The AND mask - page flags X86_PTE_*.
1012	*/
1013	PGM_GST_DECL(int, ModifyPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
1014	{
1015	Assert((cb & GUEST_PAGE_OFFSET_MASK) == 0); RT_NOREF_PV(cb);
1016
1017	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
1018	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1019	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
1020	for (;;)
1021	{
1022	PGMPTWALK Walk;
1023	GSTPTWALK GstWalk;
1024	int rc = PGM_GST_NAME(Walk)(pVCpu, GCPtr, &Walk, &GstWalk);
1025	if (RT_FAILURE(rc))
1026	return rc;
1027
1028	if (!Walk.fBigPage)
1029	{
1030	/*
1031	* 4KB Page table, process
1032	*
1033	* Walk pages till we're done.
1034	*/
1035	unsigned iPTE = (GCPtr >> GST_PT_SHIFT) & GST_PT_MASK;
1036	while (iPTE < RT_ELEMENTS(GstWalk.pPt->a))
1037	{
1038	GSTPTE Pte = GstWalk.pPt->a[iPTE];
1039	Pte.u = (Pte.u & (fMask \| X86_PTE_PAE_PG_MASK))
1040	\| (fFlags & ~GST_PTE_PG_MASK);
1041	GstWalk.pPt->a[iPTE] = Pte;
1042
1043	/* next page */
1044	cb -= GUEST_PAGE_SIZE;
1045	if (!cb)
1046	return VINF_SUCCESS;
1047	GCPtr += GUEST_PAGE_SIZE;
1048	iPTE++;
1049	}
1050	}
1051	else
1052	{
1053	/*
1054	* 2/4MB Page table
1055	*/
1056	GSTPDE PdeNew;
1057	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1058	PdeNew.u = (GstWalk.Pde.u & (fMask \| ((fMask & X86_PTE_PAT) << X86_PDE4M_PAT_SHIFT) \| GST_PDE_BIG_PG_MASK \| X86_PDE4M_PG_HIGH_MASK \| X86_PDE4M_PS))
1059	# else
1060	PdeNew.u = (GstWalk.Pde.u & (fMask \| ((fMask & X86_PTE_PAT) << X86_PDE4M_PAT_SHIFT) \| GST_PDE_BIG_PG_MASK \| X86_PDE4M_PS))
1061	# endif
1062	\| (fFlags & ~GST_PTE_PG_MASK)
1063	\| ((fFlags & X86_PTE_PAT) << X86_PDE4M_PAT_SHIFT);
1064	*GstWalk.pPde = PdeNew;
1065
1066	/* advance */
1067	const unsigned cbDone = GST_BIG_PAGE_SIZE - (GCPtr & GST_BIG_PAGE_OFFSET_MASK);
1068	if (cbDone >= cb)
1069	return VINF_SUCCESS;
1070	cb -= cbDone;
1071	GCPtr += cbDone;
1072	}
1073	}
1074
1075	#else
1076	/* real / protected mode: ignore. */
1077	NOREF(pVCpu); NOREF(GCPtr); NOREF(fFlags); NOREF(fMask);
1078	return VINF_SUCCESS;
1079	#endif
1080	}
1081
1082
1083	#ifdef IN_RING3
1084	/**
1085	* Relocate any GC pointers related to guest mode paging.
1086	*
1087	* @returns VBox status code.
1088	* @param pVCpu The cross context virtual CPU structure.
1089	* @param offDelta The relocation offset.
1090	*/
1091	PGM_GST_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta)
1092	{
1093	RT_NOREF(pVCpu, offDelta);
1094	return VINF_SUCCESS;
1095	}
1096	#endif

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllGst.h@ 105844

Download in other formats: