PGMAllGst.h@ 104935

Last change on this file since 104935 was 104935, checked in by vboxsync, 10 months ago
VMM/PGM: Lockless pgmPhysGCPhys2R3Ptr variant for use in PGMGstQueryPage. bugref:10687
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 42.0 KB

Line
1	/* $Id: PGMAllGst.h 104935 2024-06-15 01:40:08Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Guest Paging Template - All context code.
4	*/
5
6	/*
7	* Copyright (C) 2006-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Internal Functions *
31	*********************************************************************************************************************************/
32	RT_C_DECLS_BEGIN
33	/** @todo Do we really need any of these forward declarations? */
34	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
35	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
36	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
37	DECLINLINE(int) PGM_GST_NAME(Walk)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PGSTPTWALK pGstWalk);
38	#endif
39	PGM_GST_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
40	PGM_GST_DECL(int, GetPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk);
41	PGM_GST_DECL(int, ModifyPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask);
42	PGM_GST_DECL(int, Exit)(PVMCPUCC pVCpu);
43
44	#ifdef IN_RING3 /* r3 only for now. */
45	PGM_GST_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta);
46	#endif
47	RT_C_DECLS_END
48
49
50	/**
51	* Enters the guest mode.
52	*
53	* @returns VBox status code.
54	* @param pVCpu The cross context virtual CPU structure.
55	* @param GCPhysCR3 The physical address from the CR3 register.
56	*/
57	PGM_GST_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
58	{
59	/*
60	* Map and monitor CR3
61	*/
62	uintptr_t idxBth = pVCpu->pgm.s.idxBothModeData;
63	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
64	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
65	return g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
66	}
67
68
69	/**
70	* Exits the guest mode.
71	*
72	* @returns VBox status code.
73	* @param pVCpu The cross context virtual CPU structure.
74	*/
75	PGM_GST_DECL(int, Exit)(PVMCPUCC pVCpu)
76	{
77	uintptr_t idxBth = pVCpu->pgm.s.idxBothModeData;
78	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
79	AssertReturn(g_aPgmBothModeData[idxBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
80	return g_aPgmBothModeData[idxBth].pfnUnmapCR3(pVCpu);
81	}
82
83
84	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
85	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
86	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
87
88
89	DECLINLINE(int) PGM_GST_NAME(WalkReturnNotPresent)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, uint8_t uLevel)
90	{
91	NOREF(pVCpu);
92	pWalk->fNotPresent = true;
93	pWalk->uLevel = uLevel;
94	pWalk->fFailed = PGM_WALKFAIL_NOT_PRESENT
95	\| ((uint32_t)uLevel << PGM_WALKFAIL_LEVEL_SHIFT);
96	return VERR_PAGE_TABLE_NOT_PRESENT;
97	}
98
99	DECLINLINE(int) PGM_GST_NAME(WalkReturnBadPhysAddr)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, uint8_t uLevel, int rc)
100	{
101	AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); NOREF(rc); NOREF(pVCpu);
102	pWalk->fBadPhysAddr = true;
103	pWalk->uLevel = uLevel;
104	pWalk->fFailed = PGM_WALKFAIL_BAD_PHYSICAL_ADDRESS
105	\| ((uint32_t)uLevel << PGM_WALKFAIL_LEVEL_SHIFT);
106	return VERR_PAGE_TABLE_NOT_PRESENT;
107	}
108
109	DECLINLINE(int) PGM_GST_NAME(WalkReturnRsvdError)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, uint8_t uLevel)
110	{
111	NOREF(pVCpu);
112	pWalk->fRsvdError = true;
113	pWalk->uLevel = uLevel;
114	pWalk->fFailed = PGM_WALKFAIL_RESERVED_BITS
115	\| ((uint32_t)uLevel << PGM_WALKFAIL_LEVEL_SHIFT);
116	return VERR_PAGE_TABLE_NOT_PRESENT;
117	}
118
119
120	/**
121	* Performs a guest page table walk.
122	*
123	* @returns VBox status code.
124	* @retval VINF_SUCCESS on success.
125	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
126	*
127	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
128	* @param GCPtr The guest virtual address to walk by.
129	* @param pWalk The page walk info.
130	* @param pGstWalk The guest mode specific page walk info.
131	* @thread EMT(pVCpu)
132	*/
133	DECLINLINE(int) PGM_GST_NAME(Walk)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PGSTPTWALK pGstWalk)
134	{
135	int rc;
136
137	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
138	/** @def PGM_GST_SLAT_WALK
139	* Macro to perform guest second-level address translation (EPT or Nested).
140	*
141	* @param a_pVCpu The cross context virtual CPU structure of the calling
142	* EMT.
143	* @param a_GCPtrNested The nested-guest linear address that caused the
144	* second-level translation.
145	* @param a_GCPhysNested The nested-guest physical address to translate.
146	* @param a_GCPhysOut Where to store the guest-physical address (result).
147	*/
148	# define PGM_GST_SLAT_WALK(a_pVCpu, a_GCPtrNested, a_GCPhysNested, a_GCPhysOut, a_pWalk) \
149	do { \
150	if ((a_pVCpu)->pgm.s.enmGuestSlatMode == PGMSLAT_EPT) \
151	{ \
152	PGMPTWALK WalkSlat; \
153	PGMPTWALKGST WalkGstSlat; \
154	int const rcX = pgmGstSlatWalk(a_pVCpu, a_GCPhysNested, true /* fIsLinearAddrValid */, a_GCPtrNested, &WalkSlat, \
155	&WalkGstSlat); \
156	if (RT_SUCCESS(rcX)) \
157	(a_GCPhysOut) = WalkSlat.GCPhys; \
158	else \
159	{ \
160	*(a_pWalk) = WalkSlat; \
161	return rcX; \
162	} \
163	} \
164	} while (0)
165	# endif
166
167	/*
168	* Init the walking structures.
169	*/
170	RT_ZERO(*pWalk);
171	RT_ZERO(*pGstWalk);
172	pWalk->GCPtr = GCPtr;
173
174	# if PGM_GST_TYPE == PGM_TYPE_32BIT \
175	\|\| PGM_GST_TYPE == PGM_TYPE_PAE
176	/*
177	* Boundary check for PAE and 32-bit (prevents trouble further down).
178	*/
179	if (RT_UNLIKELY(GCPtr >= _4G))
180	return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 8);
181	# endif
182
183	uint64_t fEffective;
184	{
185	# if PGM_GST_TYPE == PGM_TYPE_AMD64
186	/*
187	* The PML4 table.
188	*/
189	rc = pgmGstGetLongModePML4PtrEx(pVCpu, &pGstWalk->pPml4);
190	if (RT_SUCCESS(rc)) { /* probable */ }
191	else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 4, rc);
192
193	PX86PML4E pPml4e;
194	pGstWalk->pPml4e = pPml4e = &pGstWalk->pPml4->a[(GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK];
195	X86PML4E Pml4e;
196	pGstWalk->Pml4e.u = Pml4e.u = pPml4e->u;
197
198	if (GST_IS_PGENTRY_PRESENT(pVCpu, Pml4e)) { /* probable */ }
199	else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 4);
200
201	if (RT_LIKELY(GST_IS_PML4E_VALID(pVCpu, Pml4e))) { /* likely */ }
202	else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 4);
203
204	fEffective = Pml4e.u & ( X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_PWT \| X86_PML4E_PCD \| X86_PML4E_A
205	\| X86_PML4E_NX);
206	pWalk->fEffective = fEffective;
207
208	/*
209	* The PDPT.
210	*/
211	RTGCPHYS GCPhysPdpt = Pml4e.u & X86_PML4E_PG_MASK;
212	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
213	PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPdpt, GCPhysPdpt, pWalk);
214	# endif
215	rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhysPdpt, &pGstWalk->pPdpt);
216	if (RT_SUCCESS(rc)) { /* probable */ }
217	else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 3, rc);
218
219	# elif PGM_GST_TYPE == PGM_TYPE_PAE
220	rc = pgmGstGetPaePDPTPtrEx(pVCpu, &pGstWalk->pPdpt);
221	if (RT_SUCCESS(rc)) { /* probable */ }
222	else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 8, rc);
223	# endif
224	}
225	{
226	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
227	PX86PDPE pPdpe;
228	pGstWalk->pPdpe = pPdpe = &pGstWalk->pPdpt->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK];
229	X86PDPE Pdpe;
230	pGstWalk->Pdpe.u = Pdpe.u = pPdpe->u;
231
232	if (GST_IS_PGENTRY_PRESENT(pVCpu, Pdpe)) { /* probable */ }
233	else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 3);
234
235	if (RT_LIKELY(GST_IS_PDPE_VALID(pVCpu, Pdpe))) { /* likely */ }
236	else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 3);
237
238	# if PGM_GST_TYPE == PGM_TYPE_AMD64
239	fEffective &= (Pdpe.u & ( X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US
240	\| X86_PDPE_PWT \| X86_PDPE_PCD \| X86_PDPE_A));
241	fEffective \|= Pdpe.u & X86_PDPE_LM_NX;
242	# else
243	/*
244	* NX in the legacy-mode PAE PDPE is reserved. The valid check above ensures the NX bit is not set.
245	* The RW, US, A bits MBZ in PAE PDPTE entries but must be 1 the way we compute cumulative (effective) access rights.
246	*/
247	Assert(!(Pdpe.u & X86_PDPE_LM_NX));
248	fEffective = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A
249	\| (Pdpe.u & (X86_PDPE_PWT \| X86_PDPE_PCD));
250	# endif
251	pWalk->fEffective = fEffective;
252
253	/*
254	* The PD.
255	*/
256	RTGCPHYS GCPhysPd = Pdpe.u & X86_PDPE_PG_MASK;
257	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
258	PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPd, GCPhysPd, pWalk);
259	# endif
260	rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhysPd, &pGstWalk->pPd);
261	if (RT_SUCCESS(rc)) { /* probable */ }
262	else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 2, rc);
263
264	# elif PGM_GST_TYPE == PGM_TYPE_32BIT
265	rc = pgmGstGet32bitPDPtrEx(pVCpu, &pGstWalk->pPd);
266	if (RT_SUCCESS(rc)) { /* probable */ }
267	else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 8, rc);
268	# endif
269	}
270	{
271	PGSTPDE pPde;
272	pGstWalk->pPde = pPde = &pGstWalk->pPd->a[(GCPtr >> GST_PD_SHIFT) & GST_PD_MASK];
273	GSTPDE Pde;
274	pGstWalk->Pde.u = Pde.u = pPde->u;
275	if (GST_IS_PGENTRY_PRESENT(pVCpu, Pde)) { /* probable */ }
276	else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 2);
277	if ((Pde.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu))
278	{
279	if (RT_LIKELY(GST_IS_BIG_PDE_VALID(pVCpu, Pde))) { /* likely */ }
280	else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 2);
281
282	/*
283	* We're done.
284	*/
285	# if PGM_GST_TYPE == PGM_TYPE_32BIT
286	fEffective = Pde.u & (X86_PDE4M_P \| X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PWT \| X86_PDE4M_PCD \| X86_PDE4M_A);
287	# else
288	fEffective &= Pde.u & (X86_PDE4M_P \| X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PWT \| X86_PDE4M_PCD \| X86_PDE4M_A);
289	fEffective \|= Pde.u & X86_PDE2M_PAE_NX;
290	# endif
291	fEffective \|= Pde.u & (X86_PDE4M_D \| X86_PDE4M_G);
292	fEffective \|= (Pde.u & X86_PDE4M_PAT) >> X86_PDE4M_PAT_SHIFT;
293	pWalk->fEffective = fEffective;
294	Assert(GST_IS_NX_ACTIVE(pVCpu) \|\| !(fEffective & PGM_PTATTRS_NX_MASK));
295	Assert(fEffective & PGM_PTATTRS_R_MASK);
296
297	pWalk->fBigPage = true;
298	pWalk->fSucceeded = true;
299	RTGCPHYS GCPhysPde = GST_GET_BIG_PDE_GCPHYS(pVCpu->CTX_SUFF(pVM), Pde)
300	\| (GCPtr & GST_BIG_PAGE_OFFSET_MASK);
301	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
302	PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPde, GCPhysPde, pWalk);
303	# endif
304	pWalk->GCPhys = GCPhysPde;
305	PGM_A20_APPLY_TO_VAR(pVCpu, pWalk->GCPhys); /** @todo why do we apply it here and not below?!? */
306	return VINF_SUCCESS;
307	}
308
309	if (RT_UNLIKELY(!GST_IS_PDE_VALID(pVCpu, Pde)))
310	return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 2);
311	# if PGM_GST_TYPE == PGM_TYPE_32BIT
312	fEffective = Pde.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD \| X86_PDE_A);
313	# else
314	fEffective &= Pde.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD \| X86_PDE_A);
315	fEffective \|= Pde.u & X86_PDE_PAE_NX;
316	# endif
317	pWalk->fEffective = fEffective;
318
319	/*
320	* The PT.
321	*/
322	RTGCPHYS GCPhysPt = GST_GET_PDE_GCPHYS(Pde);
323	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
324	PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPt, GCPhysPt, pWalk);
325	# endif
326	rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhysPt, &pGstWalk->pPt);
327	if (RT_SUCCESS(rc)) { /* probable */ }
328	else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 1, rc);
329	}
330	{
331	PGSTPTE pPte;
332	pGstWalk->pPte = pPte = &pGstWalk->pPt->a[(GCPtr >> GST_PT_SHIFT) & GST_PT_MASK];
333	GSTPTE Pte;
334	pGstWalk->Pte.u = Pte.u = pPte->u;
335
336	if (GST_IS_PGENTRY_PRESENT(pVCpu, Pte)) { /* probable */ }
337	else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 1);
338
339	if (RT_LIKELY(GST_IS_PTE_VALID(pVCpu, Pte))) { /* likely */ }
340	else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 1);
341
342	/*
343	* We're done.
344	*/
345	fEffective &= Pte.u & (X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_A);
346	fEffective \|= Pte.u & (X86_PTE_D \| X86_PTE_PAT \| X86_PTE_G);
347	# if PGM_GST_TYPE != PGM_TYPE_32BIT
348	fEffective \|= Pte.u & X86_PTE_PAE_NX;
349	# endif
350	pWalk->fEffective = fEffective;
351	Assert(GST_IS_NX_ACTIVE(pVCpu) \|\| !(fEffective & PGM_PTATTRS_NX_MASK));
352	Assert(fEffective & PGM_PTATTRS_R_MASK);
353
354	pWalk->fSucceeded = true;
355	RTGCPHYS GCPhysPte = GST_GET_PTE_GCPHYS(Pte)
356	\| (GCPtr & GUEST_PAGE_OFFSET_MASK);
357	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
358	PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPte, GCPhysPte, pWalk);
359	# endif
360	pWalk->GCPhys = GCPhysPte;
361	return VINF_SUCCESS;
362	}
363	}
364
365	#endif /* 32BIT, PAE, AMD64 */
366
367	/**
368	* Gets effective Guest OS page information.
369	*
370	* @returns VBox status code.
371	* @param pVCpu The cross context virtual CPU structure.
372	* @param GCPtr Guest Context virtual address of the page.
373	* @param pWalk Where to store the page walk info.
374	* @thread EMT(pVCpu)
375	*/
376	PGM_GST_DECL(int, GetPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk)
377	{
378	#if PGM_GST_TYPE == PGM_TYPE_REAL \
379	\|\| PGM_GST_TYPE == PGM_TYPE_PROT
380
381	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
382	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
383	{
384	PGMPTWALK WalkSlat;
385	PGMPTWALKGST WalkGstSlat;
386	int const rc = pgmGstSlatWalk(pVCpu, GCPtr, true /* fIsLinearAddrValid */, GCPtr, &WalkSlat, &WalkGstSlat);
387	if (RT_SUCCESS(rc))
388	{
389	RT_ZERO(*pWalk);
390	pWalk->fSucceeded = true;
391	pWalk->GCPtr = GCPtr;
392	pWalk->GCPhys = WalkSlat.GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
393	pWalk->fEffective = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US;
394	}
395	else
396	*pWalk = WalkSlat;
397	return rc;
398	}
399	# endif
400
401	/*
402	* Fake it.
403	*/
404	RT_ZERO(*pWalk);
405	pWalk->fSucceeded = true;
406	pWalk->GCPtr = GCPtr;
407	pWalk->GCPhys = GCPtr & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
408	pWalk->fEffective = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US;
409	NOREF(pVCpu);
410	return VINF_SUCCESS;
411
412	#elif PGM_GST_TYPE == PGM_TYPE_32BIT \
413	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
414	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
415
416	GSTPTWALK GstWalk;
417	int rc = PGM_GST_NAME(Walk)(pVCpu, GCPtr, pWalk, &GstWalk);
418	if (RT_FAILURE(rc))
419	return rc;
420
421	Assert(pWalk->fSucceeded);
422	Assert(pWalk->GCPtr == GCPtr);
423
424	PGMPTATTRS fFlags;
425	if (!pWalk->fBigPage)
426	fFlags = (GstWalk.Pte.u & ~(GST_PTE_PG_MASK \| X86_PTE_RW \| X86_PTE_US)) /* NX not needed */
427	\| (pWalk->fEffective & (PGM_PTATTRS_W_MASK \| PGM_PTATTRS_US_MASK))
428	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_GST_TYPE)
429	\| (pWalk->fEffective & PGM_PTATTRS_NX_MASK)
430	# endif
431	;
432	else
433	fFlags = (GstWalk.Pde.u & ~(GST_PTE_PG_MASK \| X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PS)) /* NX not needed */
434	\| (pWalk->fEffective & (PGM_PTATTRS_W_MASK \| PGM_PTATTRS_US_MASK \| PGM_PTATTRS_PAT_MASK))
435	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_GST_TYPE)
436	\| (pWalk->fEffective & PGM_PTATTRS_NX_MASK)
437	# endif
438	;
439
440	pWalk->GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
441	pWalk->fEffective = fFlags;
442	return VINF_SUCCESS;
443
444	#else
445	# error "shouldn't be here!"
446	/* something else... */
447	return VERR_NOT_SUPPORTED;
448	#endif
449	}
450
451
452	/* x x x x x x x x */
453	/* x x x x x x x x */
454
455	#if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) \|\| defined(VBOX_WITH_NESTED_HWVIRT_SVM_XXX) \|\| defined(DOXYGEN_RUNNING)
456	/** Converts regular style walk info to fast style. */
457	DECL_FORCE_INLINE(void) PGM_GST_NAME(ConvertPtWalkToFast)(PGMPTWALK const *pSrc, PPGMPTWALKFAST pDst)
458	{
459	pDst->GCPtr = pSrc->GCPtr;
460	pDst->GCPhys = pSrc->GCPhys;
461	pDst->GCPhysNested = pSrc->GCPhysNested;
462	pDst->fInfo = (pSrc->fSucceeded ? PGM_WALKINFO_SUCCEEDED : 0)
463	\| (pSrc->fIsSlat ? PGM_WALKINFO_IS_SLAT : 0)
464	\| (pSrc->fIsLinearAddrValid ? PGM_WALKINFO_IS_LINEAR_ADDR_VALID : 0);
465	pDst->fFailed = pSrc->fFailed \| ((uint32_t)pSrc->uLevel << PGM_WALKFAIL_LEVEL_SHIFT);
466	pDst->fEffective = pSrc->fEffective;
467	}
468	#endif
469
470
471	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
472	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
473	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
474
475	DECLINLINE(int) PGM_GST_NAME(WalkFastReturnNotPresent)(PVMCPUCC pVCpu, PPGMPTWALKFAST pWalk, uint8_t uLevel)
476	{
477	RT_NOREF(pVCpu);
478	pWalk->fFailed = PGM_WALKFAIL_NOT_PRESENT \| ((uint32_t)uLevel << PGM_WALKFAIL_LEVEL_SHIFT);
479	return VERR_PAGE_TABLE_NOT_PRESENT;
480	}
481
482	DECLINLINE(int) PGM_GST_NAME(WalkFastReturnBadPhysAddr)(PVMCPUCC pVCpu, PPGMPTWALKFAST pWalk, uint8_t uLevel, int rc)
483	{
484	AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); RT_NOREF(pVCpu, rc);
485	pWalk->fFailed = PGM_WALKFAIL_BAD_PHYSICAL_ADDRESS \| ((uint32_t)uLevel << PGM_WALKFAIL_LEVEL_SHIFT);
486	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
487	}
488
489	DECLINLINE(int) PGM_GST_NAME(WalkFastReturnRsvdError)(PVMCPUCC pVCpu, PPGMPTWALKFAST pWalk, uint8_t uLevel)
490	{
491	RT_NOREF(pVCpu);
492	pWalk->fFailed = PGM_WALKFAIL_RESERVED_BITS \| ((uint32_t)uLevel << PGM_WALKFAIL_LEVEL_SHIFT);
493	return VERR_RESERVED_PAGE_TABLE_BITS;
494	}
495
496	/**
497	* Performs a guest page table walk.
498	*
499	* @returns VBox status code.
500	* @retval VINF_SUCCESS on success.
501	* @retval VERR_PAGE_TABLE_NOT_PRESENT, VERR_RESERVED_PAGE_TABLE_BITS or
502	* VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS on normal failure.
503	* The failure reason is also recorded in PGMPTWALKFAST::fFailed.
504	*
505	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
506	* @param GCPtr The guest virtual address to walk by.
507	* @param fFlags PGMQPAGE_F_XXX.
508	* This is ignored when @a a_fSetFlags is @c false.
509	* @param pWalk The page walk info.
510	* @param pGstWalk The guest mode specific page walk info.
511	* @tparam a_enmGuestSlatMode The SLAT mode of the function.
512	* @tparam a_fSetFlags Whether to process @a fFlags and set accessed
513	* and dirty flags accordingly.
514	* @thread EMT(pVCpu)
515	*/
516	template<PGMSLAT const a_enmGuestSlatMode = PGMSLAT_DIRECT, bool const a_fSetFlags = false>
517	DECLINLINE(int) PGM_GST_NAME(WalkFast)(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fFlags, PPGMPTWALKFAST pWalk, PGSTPTWALK pGstWalk)
518	{
519	int rc;
520
521	# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) \|\| defined(VBOX_WITH_NESTED_HWVIRT_SVM_XXX) \|\| defined(DOXYGEN_RUNNING)
522	/** @def PGM_GST_SLAT_WALK
523	* Macro to perform guest second-level address translation (EPT or Nested).
524	*
525	* @param a_pVCpu The cross context virtual CPU structure of the calling
526	* EMT.
527	* @param a_GCPtrNested The nested-guest linear address that caused the
528	* second-level translation.
529	* @param a_GCPhysNested The nested-guest physical address to translate.
530	* @param a_fFinal Set to @a true if this is the final page table entry
531	* and effective nested page table flags should be
532	* merged into PGMPTWALKFAST::fEffective. Otherwise
533	* set to @a false and nothing done.
534	* @param a_GCPhysOut Where to store the guest-physical address (result).
535	* @param a_pWalk The @a pWalk argument to the function.
536	*/
537	# define PGM_GST_SLAT_WALK_FAST(a_pVCpu, a_GCPtrNested, a_GCPhysNested, a_fFinal, a_GCPhysOut, a_pWalk) \
538	do { \
539	/** @todo Optimize this. Among other things, WalkSlat can be eliminated. WalkGstSlat is completely pointless. */ \
540	/** @todo pass fFlags along as appropriate... */ \
541	if (a_enmGuestSlatMode != PGMSLAT_DIRECT) \
542	{ \
543	PGMPTWALK WalkSlat; \
544	PGMPTWALKGST WalkGstSlat; \
545	int rcX; \
546	if (a_enmGuestSlatMode == PGMSLAT_EPT) \
547	rcX = PGM_GST_SLAT_NAME_EPT(Walk)(a_pVCpu, a_GCPhysNested, true /* fIsLinearAddrValid */, a_GCPtrNested, \
548	&WalkSlat, &WalkGstSlat.u.Ept); \
549	else AssertFailedReturn(VERR_NOT_IMPLEMENTED); \
550	if (RT_SUCCESS(rcX)) \
551	(a_GCPhysOut) = WalkSlat.GCPhys; \
552	else \
553	{ \
554	PGM_GST_NAME(ConvertPtWalkToFast)(&WalkSlat, pWalk); \
555	return rcX; \
556	} \
557	if (a_fFinal) \
558	{ /* Merge in the nested paging flags for the final GCPhys. */ \
559	if (a_enmGuestSlatMode == PGMSLAT_EPT) \
560	(a_pWalk)->fEffective = ((a_pWalk)->fEffective & ~PGM_PTATTRS_EPT_MASK) \
561	\| (WalkSlat.fEffective & PGM_PTATTRS_EPT_MASK); \
562	else AssertFailedReturn(VERR_NOT_IMPLEMENTED); \
563	} \
564	} \
565	} while (0)
566	# else
567	# define PGM_GST_SLAT_WALK_FAST(a_pVCpu, a_GCPtrNested, a_GCPhysNested, a_fFinal, a_GCPhysOut, a_pWalk) do { } while (0)
568	# endif
569	# if PGM_GST_TYPE == PGM_TYPE_32BIT
570	# define PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, a_fEffective, a_pEntryU, a_OrgEntryU, a_fFlags) do { \
571	if (!a_fSetFlags \|\| ((a_OrgEntryU) & (a_fFlags)) == (a_fFlags)) \
572	{ /* likely */ } \
573	else \
574	{ \
575	ASMAtomicOrU32((a_pEntryU), (a_fFlags)); \
576	(a_fEffective) \|= (a_fFlags); \
577	} \
578	} while (0)
579	# else
580	# define PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, a_fEffective, a_pEntryU, a_OrgEntryU, a_fFlags) do { \
581	if (!a_fSetFlags \|\| ((a_OrgEntryU) & (a_fFlags)) == (a_fFlags)) \
582	{ /* likely */ } \
583	else \
584	{ \
585	ASMAtomicOrU64((a_pEntryU), (a_fFlags)); \
586	(a_fEffective) \|= (a_fFlags); \
587	} \
588	} while (0)
589	# endif
590
591
592	/*
593	* Init the walking structures.
594	*/
595	RT_ZERO(*pGstWalk);
596	pWalk->GCPtr = GCPtr;
597	pWalk->GCPhys = 0;
598	pWalk->GCPhysNested = 0;
599	pWalk->fInfo = 0;
600	pWalk->fFailed = 0;
601	pWalk->fEffective = 0;
602
603	# if PGM_GST_TYPE == PGM_TYPE_32BIT \
604	\|\| PGM_GST_TYPE == PGM_TYPE_PAE
605	/*
606	* Boundary check for PAE and 32-bit (prevents trouble further down).
607	*/
608	if (RT_LIKELY(GCPtr < _4G))
609	{ /* extremely likely */ }
610	else
611	return PGM_GST_NAME(WalkFastReturnNotPresent)(pVCpu, pWalk, 8);
612	# endif
613
614	uint64_t fEffective;
615	{
616	# if PGM_GST_TYPE == PGM_TYPE_AMD64
617	/*
618	* The PML4 table.
619	*/
620	rc = pgmGstGetLongModePML4PtrEx(pVCpu, &pGstWalk->pPml4);
621	if (RT_SUCCESS(rc)) { /* probable */ }
622	else return PGM_GST_NAME(WalkFastReturnBadPhysAddr)(pVCpu, pWalk, 4, rc);
623
624	PX86PML4E pPml4e;
625	pGstWalk->pPml4e = pPml4e = &pGstWalk->pPml4->a[(GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK];
626	X86PML4E Pml4e;
627	pGstWalk->Pml4e.u = Pml4e.u = ASMAtomicUoReadU64(&pPml4e->u);
628
629	if (GST_IS_PGENTRY_PRESENT(pVCpu, Pml4e)) { /* probable */ }
630	else return PGM_GST_NAME(WalkFastReturnNotPresent)(pVCpu, pWalk, 4);
631
632	if (RT_LIKELY(GST_IS_PML4E_VALID(pVCpu, Pml4e))) { /* likely */ }
633	else return PGM_GST_NAME(WalkFastReturnRsvdError)(pVCpu, pWalk, 4);
634
635	fEffective = Pml4e.u & ( X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_PWT \| X86_PML4E_PCD \| X86_PML4E_A
636	\| X86_PML4E_NX);
637	PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, fEffective, &pPml4e->u, Pml4e.u, X86_PML4E_A);
638	pWalk->fEffective = fEffective;
639
640	/*
641	* The PDPT.
642	*/
643	RTGCPHYS GCPhysPdpt = Pml4e.u & X86_PML4E_PG_MASK;
644	PGM_GST_SLAT_WALK_FAST(pVCpu, GCPtr, GCPhysPdpt, false /a_fFinal/, GCPhysPdpt, pWalk);
645	rc = pgmPhysGCPhys2CCPtrLockless(pVCpu, GCPhysPdpt, (void **)&pGstWalk->pPdpt);
646	if (RT_SUCCESS(rc)) { /* probable */ }
647	else return PGM_GST_NAME(WalkFastReturnBadPhysAddr)(pVCpu, pWalk, 3, rc);
648
649	# elif PGM_GST_TYPE == PGM_TYPE_PAE
650	rc = pgmGstGetPaePDPTPtrEx(pVCpu, &pGstWalk->pPdpt);
651	if (RT_SUCCESS(rc)) { /* probable */ }
652	else return PGM_GST_NAME(WalkFastReturnBadPhysAddr)(pVCpu, pWalk, 8, rc);
653	# endif
654	}
655	{
656	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
657	PX86PDPE pPdpe;
658	pGstWalk->pPdpe = pPdpe = &pGstWalk->pPdpt->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK];
659	X86PDPE Pdpe;
660	pGstWalk->Pdpe.u = Pdpe.u = ASMAtomicUoReadU64(&pPdpe->u);
661
662	if (GST_IS_PGENTRY_PRESENT(pVCpu, Pdpe)) { /* probable */ }
663	else return PGM_GST_NAME(WalkFastReturnNotPresent)(pVCpu, pWalk, 3);
664
665	if (RT_LIKELY(GST_IS_PDPE_VALID(pVCpu, Pdpe))) { /* likely */ }
666	else return PGM_GST_NAME(WalkFastReturnRsvdError)(pVCpu, pWalk, 3);
667
668	# if PGM_GST_TYPE == PGM_TYPE_AMD64
669	fEffective &= (Pdpe.u & ( X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US
670	\| X86_PDPE_PWT \| X86_PDPE_PCD \| X86_PDPE_A));
671	fEffective \|= Pdpe.u & X86_PDPE_LM_NX;
672	PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, fEffective, &pPdpe->u, Pdpe.u, X86_PDE_A);
673	# else
674	/*
675	* NX in the legacy-mode PAE PDPE is reserved. The valid check above ensures the NX bit is not set.
676	* The RW, US, A bits MBZ in PAE PDPTE entries but must be 1 the way we compute cumulative (effective) access rights.
677	*/
678	Assert(!(Pdpe.u & X86_PDPE_LM_NX));
679	fEffective = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A
680	\| (Pdpe.u & (X86_PDPE_PWT \| X86_PDPE_PCD));
681	# endif
682	pWalk->fEffective = fEffective;
683
684	/*
685	* The PD.
686	*/
687	RTGCPHYS GCPhysPd = Pdpe.u & X86_PDPE_PG_MASK;
688	PGM_GST_SLAT_WALK_FAST(pVCpu, GCPtr, GCPhysPd, false /a_fFinal/, GCPhysPd, pWalk);
689	rc = pgmPhysGCPhys2CCPtrLockless(pVCpu, GCPhysPd, (void **)&pGstWalk->pPd);
690	if (RT_SUCCESS(rc)) { /* probable */ }
691	else return PGM_GST_NAME(WalkFastReturnBadPhysAddr)(pVCpu, pWalk, 2, rc);
692
693	# elif PGM_GST_TYPE == PGM_TYPE_32BIT
694	rc = pgmGstGet32bitPDPtrEx(pVCpu, &pGstWalk->pPd);
695	if (RT_SUCCESS(rc)) { /* probable */ }
696	else return PGM_GST_NAME(WalkFastReturnBadPhysAddr)(pVCpu, pWalk, 8, rc);
697	# endif
698	}
699	{
700	PGSTPDE pPde;
701	pGstWalk->pPde = pPde = &pGstWalk->pPd->a[(GCPtr >> GST_PD_SHIFT) & GST_PD_MASK];
702	GSTPDE Pde;
703	# if PGM_GST_TYPE != PGM_TYPE_32BIT
704	pGstWalk->Pde.u = Pde.u = ASMAtomicUoReadU64(&pPde->u);
705	# else
706	pGstWalk->Pde.u = Pde.u = ASMAtomicUoReadU32(&pPde->u);
707	# endif
708	if (GST_IS_PGENTRY_PRESENT(pVCpu, Pde)) { /* probable */ }
709	else return PGM_GST_NAME(WalkFastReturnNotPresent)(pVCpu, pWalk, 2);
710	if ((Pde.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu))
711	{
712	if (RT_LIKELY(GST_IS_BIG_PDE_VALID(pVCpu, Pde))) { /* likely */ }
713	else return PGM_GST_NAME(WalkFastReturnRsvdError)(pVCpu, pWalk, 2);
714
715	/*
716	* We're done.
717	*/
718	pWalk->fInfo = PGM_WALKINFO_SUCCEEDED \| PGM_WALKINFO_BIG_PAGE;
719
720	# if PGM_GST_TYPE == PGM_TYPE_32BIT
721	fEffective = Pde.u & (X86_PDE4M_P \| X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PWT \| X86_PDE4M_PCD \| X86_PDE4M_A);
722	# else
723	fEffective &= Pde.u & (X86_PDE4M_P \| X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PWT \| X86_PDE4M_PCD \| X86_PDE4M_A);
724	fEffective \|= Pde.u & X86_PDE2M_PAE_NX;
725	# endif
726	fEffective \|= Pde.u & (X86_PDE4M_D \| X86_PDE4M_G);
727	fEffective \|= (Pde.u & X86_PDE4M_PAT) >> X86_PDE4M_PAT_SHIFT;
728
729	rc = VINF_SUCCESS;
730	if (a_fSetFlags)
731	{
732	/* We have to validate the access before setting any flags. */
733	uint32_t fFailed = 0;
734	if ((fFlags & PGMQPAGE_F_USER_MODE) && !(fEffective & X86_PDE4M_US))
735	fFailed \|= PGM_WALKFAIL_NOT_ACCESSIBLE_BY_MODE;
736	if (fFlags & PGMQPAGE_F_WRITE)
737	{
738	if ( (fEffective & X86_PDE4M_RW)
739	\|\| (fFlags & (PGMQPAGE_F_USER_MODE \| PGMQPAGE_F_CR0_WP0)) == PGMQPAGE_F_CR0_WP0)
740	{ /* likely*/ }
741	else fFailed \|= PGM_WALKFAIL_NOT_WRITABLE;
742	}
743	# if PGM_GST_TYPE != PGM_TYPE_32BIT
744	else if (fFlags & PGMQPAGE_F_EXECUTE)
745	{
746	if (!(fEffective & X86_PDE2M_PAE_NX) \|\| !pVCpu->pgm.s.fNoExecuteEnabled) { /* likely */ }
747	else fFailed \|= PGM_WALKFAIL_NOT_EXECUTABLE;
748	}
749	# endif
750	if (fFailed == 0)
751	{
752	if (!(fFlags & PGMQPAGE_F_WRITE))
753	PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, fEffective, &pPde->u, Pde.u, X86_PDE4M_A);
754	else
755	PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, fEffective, &pPde->u, Pde.u, X86_PDE4M_A \| X86_PDE4M_D);
756	}
757	else
758	{
759	pWalk->fFailed = fFailed \| (2U << PGM_WALKFAIL_LEVEL_SHIFT);
760	pWalk->fInfo = PGM_WALKINFO_BIG_PAGE;
761	rc = VERR_ACCESS_DENIED;
762	}
763	}
764
765	pWalk->fEffective = fEffective;
766	Assert(GST_IS_NX_ACTIVE(pVCpu) \|\| !(fEffective & PGM_PTATTRS_NX_MASK));
767	Assert(fEffective & PGM_PTATTRS_R_MASK);
768
769	RTGCPHYS GCPhysPde = GST_GET_BIG_PDE_GCPHYS(pVCpu->CTX_SUFF(pVM), Pde)
770	\| (GCPtr & GST_BIG_PAGE_OFFSET_MASK);
771	PGM_GST_SLAT_WALK_FAST(pVCpu, GCPtr, GCPhysPde, true /a_fFinal/, GCPhysPde, pWalk);
772	pWalk->GCPhys = GCPhysPde;
773	PGM_A20_APPLY_TO_VAR(pVCpu, pWalk->GCPhys); /** @todo why do we apply it here and not below?!? */
774	return rc;
775	}
776
777	if (RT_UNLIKELY(!GST_IS_PDE_VALID(pVCpu, Pde)))
778	return PGM_GST_NAME(WalkFastReturnRsvdError)(pVCpu, pWalk, 2);
779	# if PGM_GST_TYPE == PGM_TYPE_32BIT
780	fEffective = Pde.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD \| X86_PDE_A);
781	# else
782	fEffective &= Pde.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD \| X86_PDE_A);
783	fEffective \|= Pde.u & X86_PDE_PAE_NX;
784	# endif
785	PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, fEffective, &pPde->u, Pde.u, X86_PDE_A);
786	pWalk->fEffective = fEffective;
787
788	/*
789	* The PT.
790	*/
791	RTGCPHYS GCPhysPt = GST_GET_PDE_GCPHYS(Pde);
792	PGM_GST_SLAT_WALK_FAST(pVCpu, GCPtr, GCPhysPt, false /a_fFinal/, GCPhysPt, pWalk);
793	rc = pgmPhysGCPhys2CCPtrLockless(pVCpu, GCPhysPt, (void **)&pGstWalk->pPt);
794	if (RT_SUCCESS(rc)) { /* probable */ }
795	else return PGM_GST_NAME(WalkFastReturnBadPhysAddr)(pVCpu, pWalk, 1, rc);
796	}
797	{
798	PGSTPTE pPte;
799	pGstWalk->pPte = pPte = &pGstWalk->pPt->a[(GCPtr >> GST_PT_SHIFT) & GST_PT_MASK];
800	GSTPTE Pte;
801	# if PGM_GST_TYPE != PGM_TYPE_32BIT
802	pGstWalk->Pte.u = Pte.u = ASMAtomicUoReadU64(&pPte->u);
803	# else
804	pGstWalk->Pte.u = Pte.u = ASMAtomicUoReadU32(&pPte->u);
805	# endif
806
807	if (GST_IS_PGENTRY_PRESENT(pVCpu, Pte)) { /* probable */ }
808	else return PGM_GST_NAME(WalkFastReturnNotPresent)(pVCpu, pWalk, 1);
809
810	if (RT_LIKELY(GST_IS_PTE_VALID(pVCpu, Pte))) { /* likely */ }
811	else return PGM_GST_NAME(WalkFastReturnRsvdError)(pVCpu, pWalk, 1);
812
813	/*
814	* We're done.
815	*/
816	pWalk->fInfo = PGM_WALKINFO_SUCCEEDED;
817
818	fEffective &= Pte.u & (X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_A);
819	# if PGM_GST_TYPE != PGM_TYPE_32BIT
820	fEffective \|= Pte.u & (X86_PTE_D \| X86_PTE_PAT \| X86_PTE_G \| X86_PTE_PAE_NX);
821	# else
822	fEffective \|= Pte.u & (X86_PTE_D \| X86_PTE_PAT \| X86_PTE_G);
823	# endif
824
825	rc = VINF_SUCCESS;
826	if (a_fSetFlags)
827	{
828	/* We have to validate the access before setting any flags. */
829	uint32_t fFailed = 0;
830	if ((fFlags & PGMQPAGE_F_USER_MODE) && !(fEffective & X86_PTE_US))
831	fFailed \|= PGM_WALKFAIL_NOT_ACCESSIBLE_BY_MODE;
832	if (fFlags & PGMQPAGE_F_WRITE)
833	{
834	if ((fEffective & X86_PTE_RW) \|\| (fFlags & (PGMQPAGE_F_USER_MODE \| PGMQPAGE_F_CR0_WP0)) == PGMQPAGE_F_CR0_WP0)
835	{ /* likely*/ }
836	else fFailed \|= PGM_WALKFAIL_NOT_WRITABLE;
837	}
838	# if PGM_GST_TYPE != PGM_TYPE_32BIT
839	else if (fFlags & PGMQPAGE_F_EXECUTE)
840	{
841	if (!(fEffective & X86_PTE_PAE_NX) \|\| !pVCpu->pgm.s.fNoExecuteEnabled) { /* likely */ }
842	else fFailed \|= PGM_WALKFAIL_NOT_EXECUTABLE;
843	}
844	# endif
845	if (fFailed == 0)
846	{
847	if (!(fFlags & PGMQPAGE_F_WRITE))
848	PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, fEffective, &pPte->u, Pte.u, X86_PTE_A);
849	else
850	PGM_GST_ENSURE_ENTRY_FLAGS_SET(a_pVCpu, fEffective, &pPte->u, Pte.u, X86_PTE_A \| X86_PTE_D);
851	}
852	else
853	{
854	pWalk->fFailed = fFailed \| (1U << PGM_WALKFAIL_LEVEL_SHIFT);
855	pWalk->fInfo = 0;
856	rc = VERR_ACCESS_DENIED;
857	}
858	}
859
860	pWalk->fEffective = fEffective;
861	Assert(GST_IS_NX_ACTIVE(pVCpu) \|\| !(fEffective & PGM_PTATTRS_NX_MASK));
862	Assert(fEffective & PGM_PTATTRS_R_MASK);
863
864	RTGCPHYS GCPhysPte = GST_GET_PTE_GCPHYS(Pte)
865	\| (GCPtr & GUEST_PAGE_OFFSET_MASK);
866	PGM_GST_SLAT_WALK_FAST(pVCpu, GCPtr, GCPhysPte, true /a_fFinal/, GCPhysPte, pWalk);
867	pWalk->GCPhys = GCPhysPte;
868	return rc;
869	}
870	# undef PGM_GST_SLAT_WALK_FAST
871	# undef PGM_GST_ENSURE_ENTRY_FLAGS_SET
872	}
873
874	#endif /* 32BIT, PAE, AMD64 */
875
876	/**
877	* Guest virtual to guest physical + info translation, the faster and better
878	* version.
879	*
880	* @returns VBox status code.
881	* @param pVCpu The cross context virtual CPU structure.
882	* @param GCPtr Guest Context virtual address of the page.
883	* @param fFlags PGMQPAGE_F_XXX
884	* @param pWalk Where to store the page walk info.
885	* @thread EMT(pVCpu)
886	*/
887	#define PGM_GET_PAGE_F_WRITE
888	#define PGM_GET_PAGE_F_READ
889	PGM_GST_DECL(int, QueryPageFast)(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fFlags, PPGMPTWALKFAST pWalk)
890	{
891	#if PGM_GST_TYPE == PGM_TYPE_REAL \
892	\|\| PGM_GST_TYPE == PGM_TYPE_PROT
893
894	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
895	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
896	{
897	/** @todo optimize this case as well. */
898	/** @todo pass fFlags along. */
899	PGMPTWALK WalkSlat;
900	PGMPTWALKGST WalkGstSlat;
901	int const rc = pgmGstSlatWalk(pVCpu, GCPtr, true /* fIsLinearAddrValid */, GCPtr, &WalkSlat, &WalkGstSlat);
902	if (RT_SUCCESS(rc))
903	{
904	PGMPTWALKFAST_ZERO(pWalk);
905	pWalk->GCPtr = GCPtr;
906	pWalk->GCPhys = WalkSlat.GCPhys;
907	pWalk->GCPhysNested = 0;
908	pWalk->fInfo = PGM_WALKINFO_SUCCEEDED;
909	pWalk->fFailed = PGM_WALKFAIL_SUCCESS;
910	pWalk->fEffective = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D;
911	}
912	else
913	PGM_GST_NAME(ConvertPtWalkToFast)(&WalkSlat, pWalk);
914	return rc;
915	}
916	# endif
917
918	/*
919	* Fake it.
920	*/
921	pWalk->GCPtr = GCPtr;
922	pWalk->GCPhys = GCPtr;
923	pWalk->GCPhysNested = 0;
924	pWalk->fInfo = PGM_WALKINFO_SUCCEEDED;
925	pWalk->fFailed = PGM_WALKFAIL_SUCCESS;
926	pWalk->fEffective = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D;
927	RT_NOREF(pVCpu, fFlags);
928	return VINF_SUCCESS;
929
930	#elif PGM_GST_TYPE == PGM_TYPE_32BIT \
931	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
932	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
933
934	GSTPTWALK GstWalk;
935	int rc;
936	# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) \|\| defined(VBOX_WITH_NESTED_HWVIRT_SVM_XXX)
937	switch (pVCpu->pgm.s.enmGuestSlatMode)
938	{
939	case PGMSLAT_DIRECT:
940	# endif
941	if (fFlags)
942	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_DIRECT, true>(pVCpu, GCPtr, fFlags, pWalk, &GstWalk);
943	else
944	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_DIRECT, false>(pVCpu, GCPtr, 0, pWalk, &GstWalk);
945	# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) \|\| defined(VBOX_WITH_NESTED_HWVIRT_SVM_XXX)
946	break;
947	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
948	case PGMSLAT_EPT:
949	if (fFlags)
950	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_EPT, true>(pVCpu, GCPtr, fFlags, pWalk, &GstWalk);
951	else
952	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_EPT, false>(pVCpu, GCPtr, 0, pWalk, &GstWalk);
953	break;
954	# endif
955	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM_XXX
956	case PGMSLAT_32BIT:
957	if (fFlags)
958	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_32BIT, true>(pVCpu, GCPtr, fFlags, pWalk, &GstWalk);
959	else
960	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_32BIT, false>(pVCpu, GCPtr, 0, pWalk, &GstWalk);
961	break;
962	case PGMSLAT_PAE:
963	if (fFlags)
964	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_PAE, true>(pVCpu, GCPtr, fFlags, pWalk, &GstWalk);
965	else
966	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_PAE, false>(pVCpu, GCPtr, 0, pWalk, &GstWalk);
967	break;
968	case PGMSLAT_AMD64:
969	if (fFlags)
970	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_AMD64, true>(pVCpu, GCPtr, fFlags, pWalk, &GstWalk);
971	else
972	rc = PGM_GST_NAME(WalkFast)<PGMSLAT_AMD64, false>(pVCpu, GCPtr, 0, pWalk, &GstWalk);
973	break;
974	# endif
975	default:
976	AssertFailedReturn(VERR_INTERNAL_ERROR_4);
977	}
978	# endif
979	if (RT_SUCCESS(rc))
980	{
981	Assert(pWalk->fInfo & PGM_WALKINFO_SUCCEEDED);
982	Assert(pWalk->GCPtr == GCPtr);
983	Assert((pWalk->GCPhys & GUEST_PAGE_OFFSET_MASK) == (GCPtr & GUEST_PAGE_OFFSET_MASK));
984	return VINF_SUCCESS;
985	}
986	return rc;
987
988	#else
989	# error "shouldn't be here!"
990	/* something else... */
991	return VERR_NOT_SUPPORTED;
992	#endif
993	}
994
995	/* x x x x x x x x */
996	/* x x x x x x x x */
997
998	/**
999	* Modify page flags for a range of pages in the guest's tables
1000	*
1001	* The existing flags are ANDed with the fMask and ORed with the fFlags.
1002	*
1003	* @returns VBox status code.
1004	* @param pVCpu The cross context virtual CPU structure.
1005	* @param GCPtr Virtual address of the first page in the range. Page aligned!
1006	* @param cb Size (in bytes) of the page range to apply the modification to. Page aligned!
1007	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
1008	* @param fMask The AND mask - page flags X86_PTE_*.
1009	*/
1010	PGM_GST_DECL(int, ModifyPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
1011	{
1012	Assert((cb & GUEST_PAGE_OFFSET_MASK) == 0); RT_NOREF_PV(cb);
1013
1014	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
1015	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1016	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
1017	for (;;)
1018	{
1019	PGMPTWALK Walk;
1020	GSTPTWALK GstWalk;
1021	int rc = PGM_GST_NAME(Walk)(pVCpu, GCPtr, &Walk, &GstWalk);
1022	if (RT_FAILURE(rc))
1023	return rc;
1024
1025	if (!Walk.fBigPage)
1026	{
1027	/*
1028	* 4KB Page table, process
1029	*
1030	* Walk pages till we're done.
1031	*/
1032	unsigned iPTE = (GCPtr >> GST_PT_SHIFT) & GST_PT_MASK;
1033	while (iPTE < RT_ELEMENTS(GstWalk.pPt->a))
1034	{
1035	GSTPTE Pte = GstWalk.pPt->a[iPTE];
1036	Pte.u = (Pte.u & (fMask \| X86_PTE_PAE_PG_MASK))
1037	\| (fFlags & ~GST_PTE_PG_MASK);
1038	GstWalk.pPt->a[iPTE] = Pte;
1039
1040	/* next page */
1041	cb -= GUEST_PAGE_SIZE;
1042	if (!cb)
1043	return VINF_SUCCESS;
1044	GCPtr += GUEST_PAGE_SIZE;
1045	iPTE++;
1046	}
1047	}
1048	else
1049	{
1050	/*
1051	* 2/4MB Page table
1052	*/
1053	GSTPDE PdeNew;
1054	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1055	PdeNew.u = (GstWalk.Pde.u & (fMask \| ((fMask & X86_PTE_PAT) << X86_PDE4M_PAT_SHIFT) \| GST_PDE_BIG_PG_MASK \| X86_PDE4M_PG_HIGH_MASK \| X86_PDE4M_PS))
1056	# else
1057	PdeNew.u = (GstWalk.Pde.u & (fMask \| ((fMask & X86_PTE_PAT) << X86_PDE4M_PAT_SHIFT) \| GST_PDE_BIG_PG_MASK \| X86_PDE4M_PS))
1058	# endif
1059	\| (fFlags & ~GST_PTE_PG_MASK)
1060	\| ((fFlags & X86_PTE_PAT) << X86_PDE4M_PAT_SHIFT);
1061	*GstWalk.pPde = PdeNew;
1062
1063	/* advance */
1064	const unsigned cbDone = GST_BIG_PAGE_SIZE - (GCPtr & GST_BIG_PAGE_OFFSET_MASK);
1065	if (cbDone >= cb)
1066	return VINF_SUCCESS;
1067	cb -= cbDone;
1068	GCPtr += cbDone;
1069	}
1070	}
1071
1072	#else
1073	/* real / protected mode: ignore. */
1074	NOREF(pVCpu); NOREF(GCPtr); NOREF(fFlags); NOREF(fMask);
1075	return VINF_SUCCESS;
1076	#endif
1077	}
1078
1079
1080	#ifdef IN_RING3
1081	/**
1082	* Relocate any GC pointers related to guest mode paging.
1083	*
1084	* @returns VBox status code.
1085	* @param pVCpu The cross context virtual CPU structure.
1086	* @param offDelta The relocation offset.
1087	*/
1088	PGM_GST_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta)
1089	{
1090	RT_NOREF(pVCpu, offDelta);
1091	return VINF_SUCCESS;
1092	}
1093	#endif

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllGst.h@ 104935

Download in other formats: