PGMAllPhys.cpp@ 108732

Last change on this file since 108732 was 108708, checked in by vboxsync, 4 weeks ago

VMM: Introduce VMM_HOST_PAGE_SIZE_DYNAMIC, HOST_PAGE_SIZE_DYNAMIC, and HOST_PAGE_SHIFT_DYNAMIC, bugref:10391

HOST_PAGE_SIZE_DYNAMIC either resolves to HOST_PAGE_SIZE or calls RTSystemGetPageSize() on hosts where
the system page size is not known during build time (linux.arm64 for now).
HOST_PAGE_SHIFT_DYNAMIC is the same for the page shift.

This allows building VMM libraries which are agnostic to the host page size (at the cost of a slightly
larger overhead).

Currently enabled only on linux.arm64

Property svn:eol-style set to native
Property svn:keywords set to Id Revision

File size: 212.4 KB

Line
1	/* $Id: PGMAllPhys.cpp 108708 2025-03-24 09:16:43Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_PGM_PHYS
33	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
34	#ifdef IN_RING0
35	# define VBOX_VMM_TARGET_X86
36	#endif
37	#include <VBox/vmm/pgm.h>
38	#include <VBox/vmm/trpm.h>
39	#include <VBox/vmm/vmm.h>
40	#include <VBox/vmm/iem.h>
41	#include <VBox/vmm/iom.h>
42	#include <VBox/vmm/em.h>
43	#include <VBox/vmm/nem.h>
44	#include "PGMInternal.h"
45	#include <VBox/vmm/vmcc.h>
46	#include "PGMInline.h"
47	#include <VBox/param.h>
48	#include <VBox/err.h>
49	#include <iprt/assert.h>
50	#include <iprt/string.h>
51	#include <VBox/log.h>
52	#ifdef IN_RING3
53	# include <iprt/thread.h>
54	# ifdef VBOX_WITH_ONLY_PGM_NEM_MODE
55	# include <iprt/zero.h>
56	# endif
57	#elif defined(IN_RING0)
58	# include <iprt/mem.h>
59	# include <iprt/memobj.h>
60	#endif
61
62
63	/*********************************************************************************************************************************
64	* Defined Constants And Macros *
65	*********************************************************************************************************************************/
66	/** Enable the physical TLB. */
67	#define PGM_WITH_PHYS_TLB
68
69	/** @def PGM_HANDLER_PHYS_IS_VALID_STATUS
70	* Checks if valid physical access handler return code (normal handler, not PF).
71	*
72	* Checks if the given strict status code is one of the expected ones for a
73	* physical access handler in the current context.
74	*
75	* @returns true or false.
76	* @param a_rcStrict The status code.
77	* @param a_fWrite Whether it is a write or read being serviced.
78	*
79	* @remarks We wish to keep the list of statuses here as short as possible.
80	* When changing, please make sure to update the PGMPhysRead,
81	* PGMPhysWrite, PGMPhysReadGCPtr and PGMPhysWriteGCPtr docs too.
82	*/
83	#ifdef IN_RING3
84	# define PGM_HANDLER_PHYS_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
85	( (a_rcStrict) == VINF_SUCCESS \
86	\|\| (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT)
87	#elif defined(IN_RING0)
88	#define PGM_HANDLER_PHYS_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
89	( (a_rcStrict) == VINF_SUCCESS \
90	\|\| (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT \
91	\
92	\|\| (a_rcStrict) == ((a_fWrite) ? VINF_IOM_R3_MMIO_WRITE : VINF_IOM_R3_MMIO_READ) \
93	\|\| (a_rcStrict) == VINF_IOM_R3_MMIO_READ_WRITE \
94	\|\| ((a_rcStrict) == VINF_IOM_R3_MMIO_COMMIT_WRITE && (a_fWrite)) \
95	\
96	\|\| (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR \
97	\|\| (a_rcStrict) == VINF_EM_DBG_STOP \
98	\|\| (a_rcStrict) == VINF_EM_DBG_EVENT \
99	\|\| (a_rcStrict) == VINF_EM_DBG_BREAKPOINT \
100	\|\| (a_rcStrict) == VINF_EM_OFF \
101	\|\| (a_rcStrict) == VINF_EM_SUSPEND \
102	\|\| (a_rcStrict) == VINF_EM_RESET \
103	)
104	#else
105	# error "Context?"
106	#endif
107
108	/** @def PGM_HANDLER_VIRT_IS_VALID_STATUS
109	* Checks if valid virtual access handler return code (normal handler, not PF).
110	*
111	* Checks if the given strict status code is one of the expected ones for a
112	* virtual access handler in the current context.
113	*
114	* @returns true or false.
115	* @param a_rcStrict The status code.
116	* @param a_fWrite Whether it is a write or read being serviced.
117	*
118	* @remarks We wish to keep the list of statuses here as short as possible.
119	* When changing, please make sure to update the PGMPhysRead,
120	* PGMPhysWrite, PGMPhysReadGCPtr and PGMPhysWriteGCPtr docs too.
121	*/
122	#ifdef IN_RING3
123	# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
124	( (a_rcStrict) == VINF_SUCCESS \
125	\|\| (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT)
126	#elif defined(IN_RING0)
127	# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
128	(false /* no virtual handlers in ring-0! */ )
129	#else
130	# error "Context?"
131	#endif
132
133
134
135	/**
136	* Calculate the actual table size.
137	*
138	* The memory is layed out like this:
139	* - PGMPHYSHANDLERTREE (8 bytes)
140	* - Allocation bitmap (8-byte size align)
141	* - Slab of PGMPHYSHANDLER. Start is 64 byte aligned.
142	*/
143	uint32_t pgmHandlerPhysicalCalcTableSizes(uint32_t pcEntries, uint32_t pcbTreeAndBitmap)
144	{
145	/*
146	* A minimum of 64 entries and a maximum of ~64K.
147	*/
148	uint32_t cEntries = *pcEntries;
149	if (cEntries <= 64)
150	cEntries = 64;
151	else if (cEntries >= _64K)
152	cEntries = _64K;
153	else
154	cEntries = RT_ALIGN_32(cEntries, 16);
155
156	/*
157	* Do the initial calculation.
158	*/
159	uint32_t cbBitmap = RT_ALIGN_32(cEntries, 64) / 8;
160	uint32_t cbTreeAndBitmap = RT_ALIGN_32(sizeof(PGMPHYSHANDLERTREE) + cbBitmap, 64);
161	uint32_t cbTable = cEntries * sizeof(PGMPHYSHANDLER);
162	uint32_t cbTotal = cbTreeAndBitmap + cbTable;
163
164	/*
165	* Align the total and try use up extra space from that.
166	*/
167	uint32_t cbTotalAligned = RT_ALIGN_32(cbTotal, RT_MAX(HOST_PAGE_SIZE_DYNAMIC, _16K));
168	uint32_t cAvail = cbTotalAligned - cbTotal;
169	cAvail /= sizeof(PGMPHYSHANDLER);
170	if (cAvail >= 1)
171	for (;;)
172	{
173	cbBitmap = RT_ALIGN_32(cEntries, 64) / 8;
174	cbTreeAndBitmap = RT_ALIGN_32(sizeof(PGMPHYSHANDLERTREE) + cbBitmap, 64);
175	cbTable = cEntries * sizeof(PGMPHYSHANDLER);
176	cbTotal = cbTreeAndBitmap + cbTable;
177	if (cbTotal <= cbTotalAligned)
178	break;
179	cEntries--;
180	Assert(cEntries >= 16);
181	}
182
183	/*
184	* Return the result.
185	*/
186	*pcbTreeAndBitmap = cbTreeAndBitmap;
187	*pcEntries = cEntries;
188	return cbTotalAligned;
189	}
190
191
192
193	/*********************************************************************************************************************************
194	* Access Handlers for ROM and MMIO2 *
195	*********************************************************************************************************************************/
196
197	#ifndef IN_RING3
198
199	/**
200	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
201	* \#PF access handler callback for guest ROM range write access.}
202	*
203	* @remarks The @a uUser argument is the PGMROMRANGE::GCPhys value.
204	*/
205	DECLCALLBACK(VBOXSTRICTRC) pgmPhysRomWritePfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx,
206	RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
207
208	{
209	AssertReturn(uUser < RT_ELEMENTS(pVM->pgmr0.s.apRomRanges), VINF_EM_RAW_EMULATE_INSTR);
210	PPGMROMRANGE const pRom = pVM->pgmr0.s.apRomRanges[uUser];
211	AssertReturn(pRom, VINF_EM_RAW_EMULATE_INSTR);
212
213	uint32_t const iPage = (GCPhysFault - pRom->GCPhys) >> GUEST_PAGE_SHIFT;
214	AssertReturn(iPage < (pRom->cb >> GUEST_PAGE_SHIFT), VERR_INTERNAL_ERROR_3);
215	#ifdef IN_RING0
216	AssertReturn(iPage < pVM->pgmr0.s.acRomRangePages[uUser], VERR_INTERNAL_ERROR_2);
217	#endif
218
219	RT_NOREF(uErrorCode, pvFault);
220	Assert(uErrorCode & X86_TRAP_PF_RW); /* This shall not be used for read access! */
221
222	int rc;
223	switch (pRom->aPages[iPage].enmProt)
224	{
225	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
226	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
227	{
228	/*
229	* If it's a simple instruction which doesn't change the cpu state
230	* we will simply skip it. Otherwise we'll have to defer it to REM.
231	*/
232	uint32_t cbOp;
233	PDISSTATE pDis = &pVCpu->pgm.s.Dis;
234	rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbOp);
235	if ( RT_SUCCESS(rc)
236	&& pDis->uCpuMode == DISCPUMODE_32BIT /** @todo why does this matter? */
237	&& !(pDis->x86.fPrefix & (DISPREFIX_REPNE \| DISPREFIX_REP \| DISPREFIX_SEG)))
238	{
239	switch (pDis->x86.bOpCode)
240	{
241	/** @todo Find other instructions we can safely skip, possibly
242	* adding this kind of detection to DIS or EM. */
243	case OP_MOV:
244	pCtx->rip += cbOp;
245	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZGuestROMWriteHandled);
246	return VINF_SUCCESS;
247	}
248	}
249	break;
250	}
251
252	case PGMROMPROT_READ_RAM_WRITE_RAM:
253	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
254	rc = PGMHandlerPhysicalPageTempOff(pVM, pRom->GCPhys, GCPhysFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
255	AssertRC(rc);
256	break; /** @todo Must edit the shadow PT and restart the instruction, not use the interpreter! */
257
258	case PGMROMPROT_READ_ROM_WRITE_RAM:
259	/* Handle it in ring-3 because it's way easier there. */
260	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
261	break;
262
263	default:
264	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhysFault=%RGp\n",
265	pRom->aPages[iPage].enmProt, iPage, GCPhysFault),
266	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
267	}
268
269	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZGuestROMWriteUnhandled);
270	return VINF_EM_RAW_EMULATE_INSTR;
271	}
272
273	#endif /* !IN_RING3 */
274
275
276	/**
277	* @callback_method_impl{FNPGMPHYSHANDLER,
278	* Access handler callback for ROM write accesses.}
279	*
280	* @remarks The @a uUser argument is the PGMROMRANGE::GCPhys value.
281	*/
282	DECLCALLBACK(VBOXSTRICTRC)
283	pgmPhysRomWriteHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
284	PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser)
285	{
286	AssertReturn(uUser < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRomRanges), VERR_INTERNAL_ERROR_3);
287	PPGMROMRANGE const pRom = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRomRanges[uUser];
288	AssertReturn(pRom, VERR_INTERNAL_ERROR_3);
289
290	uint32_t const iPage = (GCPhys - pRom->GCPhys) >> GUEST_PAGE_SHIFT;
291	AssertReturn(iPage < (pRom->cb >> GUEST_PAGE_SHIFT), VERR_INTERNAL_ERROR_2);
292	#ifdef IN_RING0
293	AssertReturn(iPage < pVM->pgmr0.s.acRomRangePages[uUser], VERR_INTERNAL_ERROR_2);
294	#endif
295	PPGMROMPAGE const pRomPage = &pRom->aPages[iPage];
296
297	Log5(("pgmPhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
298	RT_NOREF(pVCpu, pvPhys, enmOrigin);
299
300	if (enmAccessType == PGMACCESSTYPE_READ)
301	{
302	switch (pRomPage->enmProt)
303	{
304	/*
305	* Take the default action.
306	*/
307	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
308	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
309	case PGMROMPROT_READ_ROM_WRITE_RAM:
310	case PGMROMPROT_READ_RAM_WRITE_RAM:
311	return VINF_PGM_HANDLER_DO_DEFAULT;
312
313	default:
314	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
315	pRom->aPages[iPage].enmProt, iPage, GCPhys),
316	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
317	}
318	}
319	else
320	{
321	Assert(enmAccessType == PGMACCESSTYPE_WRITE);
322	switch (pRomPage->enmProt)
323	{
324	/*
325	* Ignore writes.
326	*/
327	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
328	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
329	return VINF_SUCCESS;
330
331	/*
332	* Write to the RAM page.
333	*/
334	case PGMROMPROT_READ_ROM_WRITE_RAM:
335	case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's way simpler that way. */
336	{
337	/* This should be impossible now, pvPhys doesn't work cross page anylonger. */
338	Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> GUEST_PAGE_SHIFT) == iPage);
339
340	/*
341	* Take the lock, do lazy allocation, map the page and copy the data.
342	*
343	* Note that we have to bypass the mapping TLB since it works on
344	* guest physical addresses and entering the shadow page would
345	* kind of screw things up...
346	*/
347	PGM_LOCK_VOID(pVM);
348
349	PPGMPAGE pShadowPage = &pRomPage->Shadow;
350	if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
351	{
352	pShadowPage = pgmPhysGetPage(pVM, GCPhys);
353	AssertLogRelMsgReturnStmt(pShadowPage, ("%RGp\n", GCPhys), PGM_UNLOCK(pVM), VERR_PGM_PHYS_PAGE_GET_IPE);
354	}
355
356	void *pvDstPage;
357	int rc;
358	#if defined(VBOX_WITH_PGM_NEM_MODE) && defined(IN_RING3)
359	if (PGM_IS_IN_NEM_MODE(pVM) && PGMROMPROT_IS_ROM(pRomPage->enmProt))
360	{
361	pvDstPage = &pRom->pbR3Alternate[GCPhys - pRom->GCPhys];
362	rc = VINF_SUCCESS;
363	}
364	else
365	#endif
366	{
367	rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK, &pvDstPage);
368	if (RT_SUCCESS(rc))
369	pvDstPage = (uint8_t *)pvDstPage + (GCPhys & GUEST_PAGE_OFFSET_MASK);
370	}
371	if (RT_SUCCESS(rc))
372	{
373	memcpy((uint8_t *)pvDstPage + (GCPhys & GUEST_PAGE_OFFSET_MASK), pvBuf, cbBuf);
374	pRomPage->LiveSave.fWrittenTo = true;
375
376	AssertMsg( rc == VINF_SUCCESS
377	\|\| ( rc == VINF_PGM_SYNC_CR3
378	&& VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
379	, ("%Rrc\n", rc));
380	rc = VINF_SUCCESS;
381	}
382
383	PGM_UNLOCK(pVM);
384	return rc;
385	}
386
387	default:
388	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
389	pRom->aPages[iPage].enmProt, iPage, GCPhys),
390	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
391	}
392	}
393	}
394
395
396	/**
397	* Common worker for pgmPhysMmio2WriteHandler and pgmPhysMmio2WritePfHandler.
398	*/
399	static VBOXSTRICTRC pgmPhysMmio2WriteHandlerCommon(PVMCC pVM, PVMCPUCC pVCpu, uint64_t hMmio2, RTGCPHYS GCPhys, RTGCPTR GCPtr)
400	{
401	/*
402	* Get the MMIO2 range.
403	*/
404	AssertReturn(hMmio2 < RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges), VERR_INTERNAL_ERROR_3);
405	AssertReturn(hMmio2 != 0, VERR_INTERNAL_ERROR_3);
406	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[hMmio2 - 1];
407	Assert(pMmio2->idMmio2 == hMmio2);
408	AssertReturn((pMmio2->fFlags & PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES) == PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES,
409	VERR_INTERNAL_ERROR_4);
410
411	/*
412	* Get the page and make sure it's an MMIO2 page.
413	*/
414	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
415	AssertReturn(pPage, VINF_EM_RAW_EMULATE_INSTR);
416	AssertReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2, VINF_EM_RAW_EMULATE_INSTR);
417
418	/*
419	* Set the dirty flag so we can avoid scanning all the pages when it isn't dirty.
420	* (The PGM_PAGE_HNDL_PHYS_STATE_DISABLED handler state indicates that a single
421	* page is dirty, saving the need for additional storage (bitmap).)
422	*/
423	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_IS_DIRTY;
424
425	/*
426	* Disable the handler for this page.
427	*/
428	int rc = PGMHandlerPhysicalPageTempOff(pVM, pMmio2->GCPhys, GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
429	AssertRC(rc);
430	#ifndef IN_RING3
431	if (RT_SUCCESS(rc) && GCPtr != ~(RTGCPTR)0)
432	{
433	rc = PGMShwMakePageWritable(pVCpu, GCPtr, PGM_MK_PG_IS_MMIO2 \| PGM_MK_PG_IS_WRITE_FAULT);
434	AssertMsgReturn(rc == VINF_SUCCESS \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT,
435	("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", GCPtr, rc), rc);
436	}
437	#else
438	RT_NOREF(pVCpu, GCPtr);
439	#endif
440	return VINF_SUCCESS;
441	}
442
443
444	#ifndef IN_RING3
445	/**
446	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
447	* \#PF access handler callback for guest MMIO2 dirty page tracing.}
448	*
449	* @remarks The @a uUser is the MMIO2 index.
450	*/
451	DECLCALLBACK(VBOXSTRICTRC) pgmPhysMmio2WritePfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx,
452	RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
453	{
454	RT_NOREF(pVCpu, uErrorCode, pCtx);
455	VBOXSTRICTRC rcStrict = PGM_LOCK(pVM); /* We should already have it, but just make sure we do. */
456	if (RT_SUCCESS(rcStrict))
457	{
458	rcStrict = pgmPhysMmio2WriteHandlerCommon(pVM, pVCpu, uUser, GCPhysFault, pvFault);
459	PGM_UNLOCK(pVM);
460	}
461	return rcStrict;
462	}
463	#endif /* !IN_RING3 */
464
465
466	/**
467	* @callback_method_impl{FNPGMPHYSHANDLER,
468	* Access handler callback for MMIO2 dirty page tracing.}
469	*
470	* @remarks The @a uUser is the MMIO2 index.
471	*/
472	DECLCALLBACK(VBOXSTRICTRC)
473	pgmPhysMmio2WriteHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
474	PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser)
475	{
476	VBOXSTRICTRC rcStrict = PGM_LOCK(pVM); /* We should already have it, but just make sure we do. */
477	if (RT_SUCCESS(rcStrict))
478	{
479	rcStrict = pgmPhysMmio2WriteHandlerCommon(pVM, pVCpu, uUser, GCPhys, ~(RTGCPTR)0);
480	PGM_UNLOCK(pVM);
481	if (rcStrict == VINF_SUCCESS)
482	rcStrict = VINF_PGM_HANDLER_DO_DEFAULT;
483	}
484	RT_NOREF(pvPhys, pvBuf, cbBuf, enmAccessType, enmOrigin);
485	return rcStrict;
486	}
487
488
489
490	/*********************************************************************************************************************************
491	* RAM Ranges *
492	*********************************************************************************************************************************/
493
494	#ifdef VBOX_STRICT
495	/**
496	* Asserts that the RAM range structures are sane.
497	*/
498	DECLHIDDEN(bool) pgmPhysAssertRamRangesLocked(PVMCC pVM, bool fInUpdate, bool fRamRelaxed)
499	{
500	bool fRet = true;
501
502	/*
503	* Check the generation ID. This is stable since we own the PGM lock.
504	*/
505	AssertStmt((pVM->pgm.s.RamRangeUnion.idGeneration & 1U) == (unsigned)fInUpdate, fRet = false);
506
507	/*
508	* Check the entry count and max ID.
509	*/
510	uint32_t const idRamRangeMax = pVM->pgm.s.idRamRangeMax;
511	/* Since this is set to the highest ID, it cannot be the same as the table size. */
512	AssertStmt(idRamRangeMax < RT_ELEMENTS(pVM->pgm.s.apRamRanges), fRet = false);
513
514	/* Because ID=0 is reserved, it's one less than the table size and at most the
515	same as the max ID. */
516	uint32_t const cLookupEntries = pVM->pgm.s.RamRangeUnion.cLookupEntries;
517	AssertStmt(cLookupEntries < RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup), fRet = false);
518	AssertStmt(cLookupEntries <= idRamRangeMax, fRet = false);
519
520	/*
521	* Check the pointer table(s).
522	*/
523	/* The first entry shall be empty. */
524	AssertStmt(pVM->pgm.s.apRamRanges[0] == NULL, fRet = false);
525	# ifdef IN_RING0
526	AssertStmt(pVM->pgmr0.s.apRamRanges[0] == NULL, fRet = false);
527	AssertStmt(pVM->pgmr0.s.acRamRangePages[0] == 0, fRet = false);
528	# endif
529
530	uint32_t cMappedRanges = 0;
531	for (uint32_t idRamRange = 1; idRamRange <= idRamRangeMax; idRamRange++)
532	{
533	# ifdef IN_RING0
534	PPGMRAMRANGE const pRamRange = pVM->pgmr0.s.apRamRanges[idRamRange];
535	AssertContinueStmt(pRamRange, fRet = false);
536	AssertStmt(pVM->pgm.s.apRamRanges[idRamRange] != NIL_RTR3PTR, fRet = false);
537	AssertStmt( (pRamRange->cb >> GUEST_PAGE_SHIFT) == pVM->pgmr0.s.acRamRangePages[idRamRange]
538	\|\| ( (pRamRange->cb >> GUEST_PAGE_SHIFT) < pVM->pgmr0.s.acRamRangePages[idRamRange]
539	&& !(pRamRange->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX)),
540	fRet = false);
541	# else
542	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apRamRanges[idRamRange];
543	AssertContinueStmt(pRamRange, fRet = false);
544	# endif
545	AssertStmt(pRamRange->idRange == idRamRange, fRet = false);
546	if (pRamRange->GCPhys != NIL_RTGCPHYS)
547	{
548	cMappedRanges++;
549	AssertStmt((pRamRange->GCPhys & GUEST_PAGE_OFFSET_MASK) == 0, fRet = false);
550	AssertStmt((pRamRange->GCPhysLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK, fRet = false);
551	AssertStmt(pRamRange->GCPhysLast > pRamRange->GCPhys, fRet = false);
552	AssertStmt(pRamRange->GCPhysLast - pRamRange->GCPhys + 1U == pRamRange->cb, fRet = false);
553	}
554	else
555	{
556	AssertStmt(pRamRange->GCPhysLast == NIL_RTGCPHYS, fRet = false);
557	AssertStmt(PGM_RAM_RANGE_IS_AD_HOC(pRamRange) \|\| fRamRelaxed, fRet = false);
558	}
559	}
560
561	/*
562	* Check that the lookup table is sorted and contains the right information.
563	*/
564	AssertMsgStmt(cMappedRanges == cLookupEntries,
565	("cMappedRanges=%#x cLookupEntries=%#x\n", cMappedRanges, cLookupEntries),
566	fRet = false);
567	RTGCPHYS GCPhysPrev = ~(RTGCPHYS)0;
568	for (uint32_t idxLookup = 0; idxLookup < cLookupEntries; idxLookup++)
569	{
570	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
571	AssertContinueStmt(idRamRange > 0 && idRamRange <= idRamRangeMax, fRet = false);
572	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm,pgmr0,pgmrc).s.apRamRanges[idRamRange];
573	AssertContinueStmt(pRamRange, fRet = false);
574
575	AssertStmt(pRamRange->idRange == idRamRange, fRet = false);
576	AssertStmt(pRamRange->GCPhys == PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]),
577	fRet = false);
578	AssertStmt(pRamRange->GCPhysLast == pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast, fRet = false);
579
580	AssertStmt(pRamRange->GCPhys >= GCPhysPrev + 1U, fRet = false);
581	GCPhysPrev = pRamRange->GCPhysLast;
582	}
583
584	return fRet;
585	}
586	#endif /* VBOX_STRICT */
587
588
589	/**
590	* Invalidates the RAM range TLBs.
591	*
592	* @param pVM The cross context VM structure.
593	*/
594	void pgmPhysInvalidRamRangeTlbs(PVMCC pVM)
595	{
596	PGM_LOCK_VOID(pVM);
597
598	/* This is technically only required when freeing the PCNet MMIO2 range
599	during ancient saved state loading. The code freeing the RAM range
600	will make sure this function is called in both rings. */
601	RT_ZERO(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb);
602	VMCC_FOR_EACH_VMCPU_STMT(pVM, RT_ZERO(pVCpu->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb));
603
604	PGM_UNLOCK(pVM);
605	}
606
607
608	/**
609	* Tests if a value of type RTGCPHYS is negative if the type had been signed
610	* instead of unsigned.
611	*
612	* @returns @c true if negative, @c false if positive or zero.
613	* @param a_GCPhys The value to test.
614	* @todo Move me to iprt/types.h.
615	*/
616	#define RTGCPHYS_IS_NEGATIVE(a_GCPhys) ((a_GCPhys) & ((RTGCPHYS)1 << (sizeof(RTGCPHYS)*8 - 1)))
617
618
619	/**
620	* Slow worker for pgmPhysGetRange.
621	*
622	* @copydoc pgmPhysGetRange
623	* @note Caller owns the PGM lock.
624	*/
625	DECLHIDDEN(PPGMRAMRANGE) pgmPhysGetRangeSlow(PVMCC pVM, RTGCPHYS GCPhys)
626	{
627	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
628
629	uint32_t idxEnd = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
630	uint32_t idxStart = 0;
631	for (;;)
632	{
633	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
634	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
635	RTGCPHYS const cbEntryMinus1 = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast - GCPhysEntryFirst;
636	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
637	if (off <= cbEntryMinus1)
638	{
639	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
640	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), NULL);
641	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
642	Assert(pRamRange);
643	pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
644	return pRamRange;
645	}
646	if (RTGCPHYS_IS_NEGATIVE(off))
647	{
648	if (idxStart < idxLookup)
649	idxEnd = idxLookup;
650	else
651	break;
652	}
653	else
654	{
655	idxLookup += 1;
656	if (idxLookup < idxEnd)
657	idxStart = idxLookup;
658	else
659	break;
660	}
661	}
662	return NULL;
663	}
664
665
666	/**
667	* Slow worker for pgmPhysGetRangeAtOrAbove.
668	*
669	* @copydoc pgmPhysGetRangeAtOrAbove
670	*/
671	DECLHIDDEN(PPGMRAMRANGE) pgmPhysGetRangeAtOrAboveSlow(PVMCC pVM, RTGCPHYS GCPhys)
672	{
673	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
674
675	uint32_t idRamRangeLastLeft = UINT32_MAX;
676	uint32_t idxEnd = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
677	uint32_t idxStart = 0;
678	for (;;)
679	{
680	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
681	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
682	RTGCPHYS const cbEntryMinus1 = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast - GCPhysEntryFirst;
683	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
684	if (off <= cbEntryMinus1)
685	{
686	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
687	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), NULL);
688	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
689	Assert(pRamRange);
690	pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
691	return pRamRange;
692	}
693	if (RTGCPHYS_IS_NEGATIVE(off))
694	{
695	idRamRangeLastLeft = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
696	if (idxStart < idxLookup)
697	idxEnd = idxLookup;
698	else
699	break;
700	}
701	else
702	{
703	idxLookup += 1;
704	if (idxLookup < idxEnd)
705	idxStart = idxLookup;
706	else
707	break;
708	}
709	}
710	if (idRamRangeLastLeft != UINT32_MAX)
711	{
712	AssertReturn(idRamRangeLastLeft < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), NULL);
713	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRangeLastLeft];
714	Assert(pRamRange);
715	return pRamRange;
716	}
717	return NULL;
718	}
719
720
721	/**
722	* Slow worker for pgmPhysGetPage.
723	*
724	* @copydoc pgmPhysGetPage
725	*/
726	DECLHIDDEN(PPGMPAGE) pgmPhysGetPageSlow(PVMCC pVM, RTGCPHYS GCPhys)
727	{
728	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
729
730	uint32_t idxEnd = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
731	uint32_t idxStart = 0;
732	for (;;)
733	{
734	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
735	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
736	RTGCPHYS const cbEntryMinus1 = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast - GCPhysEntryFirst;
737	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
738	if (off <= cbEntryMinus1)
739	{
740	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
741	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), NULL);
742	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
743	AssertReturn(pRamRange, NULL);
744	pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
745
746	/* Get the page. */
747	Assert(off < pRamRange->cb);
748	RTGCPHYS const idxPage = off >> GUEST_PAGE_SHIFT;
749	#ifdef IN_RING0
750	AssertReturn(idxPage < pVM->pgmr0.s.acRamRangePages[idRamRange], NULL);
751	#endif
752	return &pRamRange->aPages[idxPage];
753	}
754	if (RTGCPHYS_IS_NEGATIVE(off))
755	{
756	if (idxStart < idxLookup)
757	idxEnd = idxLookup;
758	else
759	break;
760	}
761	else
762	{
763	idxLookup += 1;
764	if (idxLookup < idxEnd)
765	idxStart = idxLookup;
766	else
767	break;
768	}
769	}
770	return NULL;
771	}
772
773
774	/**
775	* Slow worker for pgmPhysGetPageEx.
776	*
777	* @copydoc pgmPhysGetPageEx
778	*/
779	DECLHIDDEN(int) pgmPhysGetPageExSlow(PVMCC pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
780	{
781	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
782
783	uint32_t idxEnd = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
784	uint32_t idxStart = 0;
785	for (;;)
786	{
787	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
788	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
789	RTGCPHYS const cbEntryMinus1 = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast - GCPhysEntryFirst;
790	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
791	if (off <= cbEntryMinus1)
792	{
793	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
794	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), VERR_PGM_PHYS_RAM_LOOKUP_IPE);
795	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
796	AssertReturn(pRamRange, VERR_PGM_PHYS_RAM_LOOKUP_IPE);
797	pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
798
799	/* Get the page. */
800	Assert(off < pRamRange->cb);
801	RTGCPHYS const idxPage = off >> GUEST_PAGE_SHIFT;
802	#ifdef IN_RING0
803	AssertReturn(idxPage < pVM->pgmr0.s.acRamRangePages[idRamRange], VERR_PGM_PHYS_RAM_LOOKUP_IPE);
804	#endif
805	*ppPage = &pRamRange->aPages[idxPage];
806	return VINF_SUCCESS;
807	}
808	if (RTGCPHYS_IS_NEGATIVE(off))
809	{
810	if (idxStart < idxLookup)
811	idxEnd = idxLookup;
812	else
813	break;
814	}
815	else
816	{
817	idxLookup += 1;
818	if (idxLookup < idxEnd)
819	idxStart = idxLookup;
820	else
821	break;
822	}
823	}
824
825	*ppPage = NULL;
826	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
827	}
828
829
830	/**
831	* Slow worker for pgmPhysGetPageAndRangeEx.
832	*
833	* @copydoc pgmPhysGetPageAndRangeEx
834	*/
835	DECLHIDDEN(int) pgmPhysGetPageAndRangeExSlow(PVMCC pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
836	{
837	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
838
839	uint32_t idxEnd = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
840	uint32_t idxStart = 0;
841	for (;;)
842	{
843	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
844	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
845	RTGCPHYS const cbEntryMinus1 = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast - GCPhysEntryFirst;
846	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
847	if (off <= cbEntryMinus1)
848	{
849	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
850	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), VERR_PGM_PHYS_RAM_LOOKUP_IPE);
851	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
852	AssertReturn(pRamRange, VERR_PGM_PHYS_RAM_LOOKUP_IPE);
853	pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
854
855	/* Get the page. */
856	Assert(off < pRamRange->cb);
857	RTGCPHYS const idxPage = off >> GUEST_PAGE_SHIFT;
858	#ifdef IN_RING0
859	AssertReturn(idxPage < pVM->pgmr0.s.acRamRangePages[idRamRange], VERR_PGM_PHYS_RAM_LOOKUP_IPE);
860	#endif
861	*ppRam = pRamRange;
862	*ppPage = &pRamRange->aPages[idxPage];
863	return VINF_SUCCESS;
864	}
865	if (RTGCPHYS_IS_NEGATIVE(off))
866	{
867	if (idxStart < idxLookup)
868	idxEnd = idxLookup;
869	else
870	break;
871	}
872	else
873	{
874	idxLookup += 1;
875	if (idxLookup < idxEnd)
876	idxStart = idxLookup;
877	else
878	break;
879	}
880	}
881
882	*ppRam = NULL;
883	*ppPage = NULL;
884	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
885	}
886
887
888	/**
889	* Slow worker for pgmPhysGetPageAndRangeExLockless.
890	*
891	* @copydoc pgmPhysGetPageAndRangeExLockless
892	*/
893	DECLHIDDEN(int) pgmPhysGetPageAndRangeExSlowLockless(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys,
894	PGMPAGE volatile ppPage, PGMRAMRANGE volatile ppRam)
895	{
896	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,RamRangeTlbMisses));
897
898	PGM::PGMRAMRANGEGENANDLOOKUPCOUNT RamRangeUnion;
899	RamRangeUnion.u64Combined = ASMAtomicUoReadU64(&pVM->pgm.s.RamRangeUnion.u64Combined);
900
901	uint32_t idxEnd = RT_MIN(RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
902	uint32_t idxStart = 0;
903	for (;;)
904	{
905	/* Read the entry as atomically as possible: */
906	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
907	PGMRAMRANGELOOKUPENTRY Entry;
908	#if (RTASM_HAVE_READ_U128+0) & 1
909	Entry.u128Normal = ASMAtomicUoReadU128U(&pVM->pgm.s.aRamRangeLookup[idxLookup].u128Volatile);
910	#else
911	Entry.u128Normal.s.Lo = pVM->pgm.s.aRamRangeLookup[idxLookup].u128Volatile.s.Lo;
912	Entry.u128Normal.s.Hi = pVM->pgm.s.aRamRangeLookup[idxLookup].u128Volatile.s.Hi;
913	ASMCompilerBarrier(); /paranoia^2/
914	if (RT_LIKELY(Entry.u128Normal.s.Lo == pVM->pgm.s.aRamRangeLookup[idxLookup].u128Volatile.s.Lo))
915	{ /* likely */ }
916	else
917	break;
918	#endif
919
920	/* Check how GCPhys relates to the entry: */
921	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(Entry);
922	RTGCPHYS const cbEntryMinus1 = Entry.GCPhysLast - GCPhysEntryFirst;
923	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
924	if (off <= cbEntryMinus1)
925	{
926	/* We seem to have a match. If, however, anything doesn't match up
927	bail and redo owning the lock. No asserting here as we may be
928	racing removal/insertion. */
929	if (!RTGCPHYS_IS_NEGATIVE(off))
930	{
931	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(Entry);
932	if (idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges))
933	{
934	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
935	if (pRamRange)
936	{
937	if ( pRamRange->GCPhys == GCPhysEntryFirst
938	&& pRamRange->cb == cbEntryMinus1 + 1U)
939	{
940	RTGCPHYS const idxPage = off >> GUEST_PAGE_SHIFT;
941	#ifdef IN_RING0
942	if (idxPage < pVM->pgmr0.s.acRamRangePages[idRamRange])
943	#endif
944	{
945	pVCpu->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
946	*ppRam = pRamRange;
947	*ppPage = &pRamRange->aPages[idxPage];
948	return VINF_SUCCESS;
949	}
950	}
951	}
952	}
953	}
954	break;
955	}
956	if (RTGCPHYS_IS_NEGATIVE(off))
957	{
958	if (idxStart < idxLookup)
959	idxEnd = idxLookup;
960	else
961	break;
962	}
963	else
964	{
965	idxLookup += 1;
966	if (idxLookup < idxEnd)
967	idxStart = idxLookup;
968	else
969	break;
970	}
971	}
972
973	/*
974	* If we get down here, we do the lookup again but while owning the PGM lock.
975	*/
976	*ppRam = NULL;
977	*ppPage = NULL;
978	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,RamRangeTlbLocking));
979
980	PGM_LOCK_VOID(pVM);
981	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, (PPGMPAGE )ppPage, (PPGMRAMRANGE )ppRam);
982	PGM_UNLOCK(pVM);
983
984	PGMRAMRANGE volatile * const pRam = *ppRam;
985	if (pRam)
986	pVCpu->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = (PPGMRAMRANGE)pRam;
987	return rc;
988	}
989
990
991	/**
992	* Common worker for pgmR3PhysAllocateRamRange, PGMR0PhysAllocateRamRangeReq,
993	* and pgmPhysMmio2RegisterWorker2.
994	*/
995	DECLHIDDEN(int) pgmPhysRamRangeAllocCommon(PVMCC pVM, uint32_t cPages, uint32_t fFlags, uint32_t *pidNewRange)
996	{
997
998	/*
999	* Allocate the RAM range structure and map it into ring-3.
1000	*/
1001	size_t const cbRamRange = RT_ALIGN_Z(RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]), HOST_PAGE_SIZE_DYNAMIC);
1002	#ifdef IN_RING0
1003	RTR0MEMOBJ hMemObj = NIL_RTR0MEMOBJ;
1004	int rc = RTR0MemObjAllocPage(&hMemObj, cbRamRange, false /fExecutable/);
1005	#else
1006	PPGMRAMRANGE pRamRange;
1007	int rc = SUPR3PageAlloc(cbRamRange >> HOST_PAGE_SHIFT_DYNAMIC, 0 /fFlags/, (void **)&pRamRange);
1008	#endif
1009	if (RT_SUCCESS(rc))
1010	{
1011	/* Zero the memory and do basic range init before mapping it into userland. */
1012	#ifdef IN_RING0
1013	PPGMRAMRANGE const pRamRange = (PPGMRAMRANGE)RTR0MemObjAddress(hMemObj);
1014	if (!RTR0MemObjWasZeroInitialized(hMemObj))
1015	#endif
1016	RT_BZERO(pRamRange, cbRamRange);
1017
1018	pRamRange->GCPhys = NIL_RTGCPHYS;
1019	pRamRange->cb = (RTGCPHYS)cPages << GUEST_PAGE_SHIFT;
1020	pRamRange->GCPhysLast = NIL_RTGCPHYS;
1021	pRamRange->fFlags = fFlags;
1022	pRamRange->idRange = UINT32_MAX / 2;
1023
1024	#ifdef IN_RING0
1025	/* Map it into userland. */
1026	RTR0MEMOBJ hMapObj = NIL_RTR0MEMOBJ;
1027	rc = RTR0MemObjMapUser(&hMapObj, hMemObj, (RTR3PTR)-1, 0 /uAlignment/,
1028	RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
1029	if (RT_SUCCESS(rc))
1030	#endif
1031	{
1032	/*
1033	* Grab the lock (unlikely to fail or block as caller typically owns it already).
1034	*/
1035	rc = PGM_LOCK(pVM);
1036	if (RT_SUCCESS(rc))
1037	{
1038	/*
1039	* Allocate a range ID.
1040	*/
1041	uint32_t idRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.idRamRangeMax + 1;
1042	if (idRamRange != 0 && idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges))
1043	{
1044	#ifdef IN_RING0
1045	if (pVM->pgmr0.s.apRamRanges[idRamRange] == NULL)
1046	#endif
1047	{
1048	if (pVM->pgm.s.apRamRanges[idRamRange] == NIL_RTR3PTR)
1049	{
1050	/*
1051	* Commit it.
1052	*/
1053	#ifdef IN_RING0
1054	pVM->pgmr0.s.apRamRanges[idRamRange] = pRamRange;
1055	pVM->pgmr0.s.acRamRangePages[idRamRange] = cPages;
1056	pVM->pgmr0.s.ahRamRangeMemObjs[idRamRange] = hMemObj;
1057	pVM->pgmr0.s.ahRamRangeMapObjs[idRamRange] = hMapObj;
1058	pVM->pgmr0.s.idRamRangeMax = idRamRange;
1059	#endif
1060
1061	pVM->pgm.s.idRamRangeMax = idRamRange;
1062	#ifdef IN_RING0
1063	pVM->pgm.s.apRamRanges[idRamRange] = RTR0MemObjAddressR3(hMapObj);
1064	#else
1065	pVM->pgm.s.apRamRanges[idRamRange] = pRamRange;
1066	#endif
1067
1068	pRamRange->idRange = idRamRange;
1069	*pidNewRange = idRamRange;
1070
1071	PGM_UNLOCK(pVM);
1072	return VINF_SUCCESS;
1073	}
1074	}
1075
1076	/*
1077	* Bail out.
1078	*/
1079	rc = VERR_INTERNAL_ERROR_5;
1080	}
1081	else
1082	rc = VERR_PGM_TOO_MANY_RAM_RANGES;
1083	PGM_UNLOCK(pVM);
1084	}
1085	#ifdef IN_RING0
1086	RTR0MemObjFree(hMapObj, false /fFreeMappings/);
1087	#endif
1088	}
1089	#ifdef IN_RING0
1090	RTR0MemObjFree(hMemObj, true /fFreeMappings/);
1091	#else
1092	SUPR3PageFree(pRamRange, cbRamRange >> HOST_PAGE_SHIFT_DYNAMIC);
1093	#endif
1094	}
1095	*pidNewRange = UINT32_MAX;
1096	return rc;
1097	}
1098
1099
1100	#ifdef IN_RING0
1101	/**
1102	* This is called during VM initialization to allocate a RAM range.
1103	*
1104	* The range is not entered into the lookup table, that is something the caller
1105	* has to do. The PGMPAGE entries are zero'ed, but otherwise uninitialized.
1106	*
1107	* @returns VBox status code.
1108	* @param pGVM Pointer to the global VM structure.
1109	* @param pReq Where to get the parameters and return the range ID.
1110	* @thread EMT(0)
1111	*/
1112	VMMR0_INT_DECL(int) PGMR0PhysAllocateRamRangeReq(PGVM pGVM, PPGMPHYSALLOCATERAMRANGEREQ pReq)
1113	{
1114	/*
1115	* Validate input (ASSUME pReq is a copy and can't be modified by ring-3
1116	* while we're here).
1117	*/
1118	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
1119	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x < %#zx\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
1120
1121	AssertReturn(pReq->cbGuestPage == GUEST_PAGE_SIZE, VERR_INCOMPATIBLE_CONFIG);
1122
1123	AssertReturn(pReq->cGuestPages > 0, VERR_OUT_OF_RANGE);
1124	AssertReturn(pReq->cGuestPages <= PGM_MAX_PAGES_PER_RAM_RANGE, VERR_OUT_OF_RANGE);
1125
1126	AssertMsgReturn(!(pReq->fFlags & ~(uint32_t)PGM_RAM_RANGE_FLAGS_VALID_MASK), ("fFlags=%#RX32\n", pReq->fFlags),
1127	VERR_INVALID_FLAGS);
1128
1129	/** @todo better VM state guard, enmVMState is ring-3 writable. */
1130	VMSTATE const enmState = pGVM->enmVMState;
1131	AssertMsgReturn(enmState == VMSTATE_CREATING, ("enmState=%d\n", enmState), VERR_VM_INVALID_VM_STATE);
1132	VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
1133
1134	/*
1135	* Call common worker.
1136	*/
1137	return pgmPhysRamRangeAllocCommon(pGVM, pReq->cGuestPages, pReq->fFlags, &pReq->idNewRange);
1138	}
1139	#endif /* IN_RING0 */
1140
1141
1142	/**
1143	* Frees a RAM range.
1144	*
1145	* This is not a typical occurence. Currently only used for a special MMIO2
1146	* saved state compatibility scenario involving PCNet and state saved before
1147	* VBox v4.3.6.
1148	*/
1149	static int pgmPhysRamRangeFree(PVMCC pVM, PPGMRAMRANGE pRamRange)
1150	{
1151	/*
1152	* Some basic input validation.
1153	*/
1154	AssertPtrReturn(pRamRange, VERR_INVALID_PARAMETER);
1155	uint32_t const idRamRange = ASMAtomicReadU32(&pRamRange->idRange);
1156	ASMCompilerBarrier();
1157	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), VERR_INVALID_PARAMETER);
1158	AssertReturn(pRamRange == pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange], VERR_INVALID_PARAMETER);
1159	AssertReturn(pRamRange->GCPhys == NIL_RTGCPHYS, VERR_RESOURCE_BUSY);
1160
1161	/*
1162	* Kill the range pointers and associated data.
1163	*/
1164	pVM->pgm.s.apRamRanges[idRamRange] = NIL_RTR3PTR;
1165	#ifdef IN_RING0
1166	pVM->pgmr0.s.apRamRanges[idRamRange] = NULL;
1167	#endif
1168
1169	/*
1170	* Zap the pages and other RAM ranges properties to ensure there aren't any
1171	* stale references to anything hanging around should the freeing go awry.
1172	*/
1173	#ifdef IN_RING0
1174	uint32_t const cPages = pVM->pgmr0.s.acRamRangePages[idRamRange];
1175	pVM->pgmr0.s.acRamRangePages[idRamRange] = 0;
1176	#else
1177	uint32_t const cPages = pRamRange->cb >> GUEST_PAGE_SHIFT;
1178	#endif
1179	RT_BZERO(pRamRange->aPages, cPages * sizeof(pRamRange->aPages[0]));
1180
1181	pRamRange->fFlags = UINT32_MAX;
1182	pRamRange->cb = NIL_RTGCPHYS;
1183	pRamRange->pbR3 = NIL_RTR3PTR;
1184	pRamRange->pszDesc = NIL_RTR3PTR;
1185	pRamRange->paLSPages = NIL_RTR3PTR;
1186	pRamRange->idRange = UINT32_MAX / 8;
1187
1188	/*
1189	* Free the RAM range itself.
1190	*/
1191	#ifdef IN_RING0
1192	Assert(pVM->pgmr0.s.ahRamRangeMapObjs[idRamRange] != NIL_RTR0MEMOBJ);
1193	int rc = RTR0MemObjFree(pVM->pgmr0.s.ahRamRangeMapObjs[idRamRange], true /fFreeMappings/);
1194	if (RT_SUCCESS(rc))
1195	{
1196	pVM->pgmr0.s.ahRamRangeMapObjs[idRamRange] = NIL_RTR0MEMOBJ;
1197	rc = RTR0MemObjFree(pVM->pgmr0.s.ahRamRangeMemObjs[idRamRange], true /fFreeMappings/);
1198	if (RT_SUCCESS(rc))
1199	pVM->pgmr0.s.ahRamRangeMemObjs[idRamRange] = NIL_RTR0MEMOBJ;
1200	}
1201	#else
1202	size_t const cbRamRange = RT_ALIGN_Z(RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]), HOST_PAGE_SIZE_DYNAMIC);
1203	int rc = SUPR3PageFree(pRamRange, cbRamRange >> HOST_PAGE_SHIFT_DYNAMIC);
1204	#endif
1205
1206	/*
1207	* Decrease the max ID if removal was successful and this was the final
1208	* RAM range entry.
1209	*/
1210	if ( RT_SUCCESS(rc)
1211	&& idRamRange == pVM->CTX_EXPR(pgm, pgmr0, pgm).s.idRamRangeMax)
1212	{
1213	pVM->pgm.s.idRamRangeMax = idRamRange - 1;
1214	#ifdef IN_RING0
1215	pVM->pgmr0.s.idRamRangeMax = idRamRange - 1;
1216	#endif
1217	}
1218
1219	/*
1220	* Make sure the RAM range TLB does not contain any stale pointers to this range.
1221	*/
1222	pgmPhysInvalidRamRangeTlbs(pVM);
1223	return rc;
1224	}
1225
1226
1227
1228	/*********************************************************************************************************************************
1229	* MMIO2 *
1230	*********************************************************************************************************************************/
1231
1232	/**
1233	* Calculates the number of chunks
1234	*
1235	* @returns Number of registration chunk needed.
1236	* @param cb The size of the MMIO/MMIO2 range.
1237	* @param pcPagesPerChunk Where to return the number of guest pages tracked by
1238	* each chunk. Optional.
1239	*/
1240	DECLHIDDEN(uint16_t) pgmPhysMmio2CalcChunkCount(RTGCPHYS cb, uint32_t *pcPagesPerChunk)
1241	{
1242	/*
1243	* This is the same calculation as PGMR3PhysRegisterRam does, except we'll be
1244	* needing a few bytes extra the PGMREGMMIO2RANGE structure.
1245	*
1246	* Note! In additions, we've got a 24 bit sub-page range for MMIO2 ranges, leaving
1247	* us with an absolute maximum of 16777215 pages per chunk (close to 64 GB).
1248	*/
1249	AssertCompile(PGM_MAX_PAGES_PER_RAM_RANGE < _16M);
1250	uint32_t const cPagesPerChunk = PGM_MAX_PAGES_PER_RAM_RANGE;
1251
1252	if (pcPagesPerChunk)
1253	*pcPagesPerChunk = cPagesPerChunk;
1254
1255	/* Calc the number of chunks we need. */
1256	RTGCPHYS const cGuestPages = cb >> GUEST_PAGE_SHIFT;
1257	uint16_t cChunks = (uint16_t)((cGuestPages + cPagesPerChunk - 1) / cPagesPerChunk);
1258	#ifdef IN_RING3
1259	AssertRelease((RTGCPHYS)cChunks * cPagesPerChunk >= cGuestPages);
1260	#else
1261	AssertReturn((RTGCPHYS)cChunks * cPagesPerChunk >= cGuestPages, 0);
1262	#endif
1263	return cChunks;
1264	}
1265
1266
1267	/**
1268	* Worker for PGMR3PhysMmio2Register and PGMR0PhysMmio2RegisterReq.
1269	*
1270	* (The caller already know which MMIO2 region ID will be assigned and how many
1271	* chunks will be used, so no output parameters required.)
1272	*/
1273	DECLHIDDEN(int) pgmPhysMmio2RegisterWorker(PVMCC pVM, uint32_t const cGuestPages, uint8_t const idMmio2,
1274	const uint8_t cChunks, PPDMDEVINSR3 const pDevIns, uint8_t
1275	const iSubDev, uint8_t const iRegion, uint32_t const fFlags)
1276	{
1277	/*
1278	* Get the number of pages per chunk.
1279	*/
1280	uint32_t cGuestPagesPerChunk;
1281	AssertReturn(pgmPhysMmio2CalcChunkCount((RTGCPHYS)cGuestPages << GUEST_PAGE_SHIFT, &cGuestPagesPerChunk) == cChunks,
1282	VERR_PGM_PHYS_MMIO_EX_IPE);
1283	Assert(idMmio2 != 0);
1284
1285	/*
1286	* The first thing we need to do is the allocate the memory that will be
1287	* backing the whole range.
1288	*/
1289	RTGCPHYS const cbMmio2Backing = (RTGCPHYS)cGuestPages << GUEST_PAGE_SHIFT;
1290	uint32_t const cHostPages = (cbMmio2Backing + HOST_PAGE_SIZE_DYNAMIC - 1U) >> HOST_PAGE_SHIFT_DYNAMIC;
1291	size_t const cbMmio2Aligned = cHostPages << HOST_PAGE_SHIFT_DYNAMIC;
1292	R3PTRTYPE(uint8_t *) pbMmio2BackingR3 = NIL_RTR3PTR;
1293	#ifdef IN_RING0
1294	RTR0MEMOBJ hMemObj = NIL_RTR0MEMOBJ;
1295	# ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
1296	int rc = RTR0MemObjAllocPage(&hMemObj, cbMmio2Aligned, false /fExecutable/);
1297	# else
1298	int rc = RTR0MemObjAllocPhysNC(&hMemObj, cbMmio2Aligned, NIL_RTHCPHYS);
1299	# endif
1300	#else /* !IN_RING0 */
1301	AssertReturn(PGM_IS_IN_NEM_MODE(pVM), VERR_INTERNAL_ERROR_4);
1302	int rc = SUPR3PageAlloc(cHostPages, pVM->pgm.s.fUseLargePages ? SUP_PAGE_ALLOC_F_LARGE_PAGES : 0, (void **)&pbMmio2BackingR3);
1303	#endif /* !IN_RING0 */
1304	if (RT_SUCCESS(rc))
1305	{
1306	/*
1307	* Make sure it's is initialized to zeros before it's mapped to userland.
1308	*/
1309	#ifdef IN_RING0
1310	# ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
1311	uint8_t pbMmio2BackingR0 = (uint8_t )RTR0MemObjAddress(hMemObj);
1312	AssertPtr(pbMmio2BackingR0);
1313	# endif
1314	rc = RTR0MemObjZeroInitialize(hMemObj, false /fForce/);
1315	AssertRCReturnStmt(rc, RTR0MemObjFree(hMemObj, true /fFreeMappings/), rc);
1316	#else
1317	RT_BZERO(pbMmio2BackingR3, cbMmio2Aligned);
1318	#endif
1319
1320	#ifdef IN_RING0
1321	/*
1322	* Map it into ring-3.
1323	*/
1324	RTR0MEMOBJ hMapObj = NIL_RTR0MEMOBJ;
1325	rc = RTR0MemObjMapUser(&hMapObj, hMemObj, (RTR3PTR)-1, 0, RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
1326	if (RT_SUCCESS(rc))
1327	{
1328	pbMmio2BackingR3 = RTR0MemObjAddressR3(hMapObj);
1329	#endif
1330
1331	/*
1332	* Create the MMIO2 registration records and associated RAM ranges.
1333	* The RAM range allocation may fail here.
1334	*/
1335	RTGCPHYS offMmio2Backing = 0;
1336	uint32_t cGuestPagesLeft = cGuestPages;
1337	for (uint32_t iChunk = 0, idx = idMmio2 - 1; iChunk < cChunks; iChunk++, idx++)
1338	{
1339	uint32_t const cPagesTrackedByChunk = RT_MIN(cGuestPagesLeft, cGuestPagesPerChunk);
1340
1341	/*
1342	* Allocate the RAM range for this chunk.
1343	*/
1344	uint32_t idRamRange = UINT32_MAX;
1345	rc = pgmPhysRamRangeAllocCommon(pVM, cPagesTrackedByChunk, PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX, &idRamRange);
1346	if (RT_FAILURE(rc))
1347	{
1348	/* We only zap the pointers to the backing storage.
1349	PGMR3Term and friends will clean up the RAM ranges and stuff. */
1350	while (iChunk-- > 0)
1351	{
1352	idx--;
1353	#ifdef IN_RING0
1354	pVM->pgmr0.s.acMmio2RangePages[idx] = 0;
1355	# ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
1356	pVM->pgmr0.s.apbMmio2Backing[idx] = NULL;
1357	# endif
1358	#endif
1359
1360	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
1361	pMmio2->pbR3 = NIL_RTR3PTR;
1362
1363	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apMmio2RamRanges[idx];
1364	pRamRange->pbR3 = NIL_RTR3PTR;
1365	RT_BZERO(&pRamRange->aPages[0], sizeof(pRamRange->aPages) * cGuestPagesPerChunk);
1366	}
1367	break;
1368	}
1369
1370	pVM->pgm.s.apMmio2RamRanges[idx] = pVM->pgm.s.apRamRanges[idRamRange];
1371	#ifdef IN_RING0
1372	pVM->pgmr0.s.apMmio2RamRanges[idx] = pVM->pgmr0.s.apRamRanges[idRamRange];
1373	pVM->pgmr0.s.acMmio2RangePages[idx] = cPagesTrackedByChunk;
1374	#endif
1375
1376	/* Initialize the RAM range. */
1377	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
1378	pRamRange->pbR3 = pbMmio2BackingR3 + offMmio2Backing;
1379	uint32_t iDstPage = cPagesTrackedByChunk;
1380	#ifdef IN_RING0
1381	AssertRelease(HOST_PAGE_SHIFT == GUEST_PAGE_SHIFT);
1382	while (iDstPage-- > 0)
1383	{
1384	RTHCPHYS HCPhys = RTR0MemObjGetPagePhysAddr(hMemObj, iDstPage + (offMmio2Backing >> HOST_PAGE_SHIFT));
1385	Assert(HCPhys != NIL_RTHCPHYS);
1386	PGM_PAGE_INIT(&pRamRange->aPages[iDstPage], HCPhys, PGM_MMIO2_PAGEID_MAKE(idMmio2, iDstPage),
1387	PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
1388	}
1389	#else
1390	Assert(PGM_IS_IN_NEM_MODE(pVM));
1391	while (iDstPage-- > 0)
1392	PGM_PAGE_INIT(&pRamRange->aPages[iDstPage], UINT64_C(0x0000ffffffff0000),
1393	PGM_MMIO2_PAGEID_MAKE(idMmio2, iDstPage),
1394	PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
1395	#endif
1396
1397	/*
1398	* Initialize the MMIO2 registration structure.
1399	*/
1400	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
1401	pMmio2->pDevInsR3 = pDevIns;
1402	pMmio2->pbR3 = pbMmio2BackingR3 + offMmio2Backing;
1403	pMmio2->fFlags = 0;
1404	if (iChunk == 0)
1405	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_FIRST_CHUNK;
1406	if (iChunk + 1 == cChunks)
1407	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_LAST_CHUNK;
1408	if (fFlags & PGMPHYS_MMIO2_FLAGS_TRACK_DIRTY_PAGES)
1409	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES;
1410
1411	pMmio2->iSubDev = iSubDev;
1412	pMmio2->iRegion = iRegion;
1413	pMmio2->idSavedState = UINT8_MAX;
1414	pMmio2->idMmio2 = idMmio2 + iChunk;
1415	pMmio2->idRamRange = idRamRange;
1416	Assert(pMmio2->idRamRange == idRamRange);
1417	pMmio2->GCPhys = NIL_RTGCPHYS;
1418	pMmio2->cbReal = (RTGCPHYS)cPagesTrackedByChunk << GUEST_PAGE_SHIFT;
1419	pMmio2->pPhysHandlerR3 = NIL_RTR3PTR; /* Pre-alloc is done by ring-3 caller. */
1420	pMmio2->paLSPages = NIL_RTR3PTR;
1421
1422	#if defined(IN_RING0) && !defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
1423	pVM->pgmr0.s.apbMmio2Backing[idx] = &pbMmio2BackingR0[offMmio2Backing];
1424	#endif
1425
1426	/* Advance */
1427	cGuestPagesLeft -= cPagesTrackedByChunk;
1428	offMmio2Backing += (RTGCPHYS)cPagesTrackedByChunk << GUEST_PAGE_SHIFT;
1429	} /* chunk alloc loop */
1430	Assert(cGuestPagesLeft == 0 \|\| RT_FAILURE_NP(rc));
1431	if (RT_SUCCESS(rc))
1432	{
1433	/*
1434	* Account for pages and ring-0 memory objects.
1435	*/
1436	pVM->pgm.s.cAllPages += cGuestPages;
1437	pVM->pgm.s.cPrivatePages += cGuestPages;
1438	#ifdef IN_RING0
1439	pVM->pgmr0.s.ahMmio2MemObjs[idMmio2 - 1] = hMemObj;
1440	pVM->pgmr0.s.ahMmio2MapObjs[idMmio2 - 1] = hMapObj;
1441	#endif
1442	pVM->pgm.s.cMmio2Ranges = idMmio2 + cChunks - 1U;
1443
1444	/*
1445	* Done!.
1446	*/
1447	return VINF_SUCCESS;
1448	}
1449
1450	/*
1451	* Bail.
1452	*/
1453	#ifdef IN_RING0
1454	RTR0MemObjFree(hMapObj, true /fFreeMappings/);
1455	}
1456	RTR0MemObjFree(hMemObj, true /fFreeMappings/);
1457	#else
1458	SUPR3PageFree(pbMmio2BackingR3, cHostPages);
1459	#endif
1460	}
1461	else
1462	LogRel(("pgmPhysMmio2RegisterWorker: Failed to allocate %RGp bytes of MMIO2 backing memory: %Rrc\n", cbMmio2Aligned, rc));
1463	return rc;
1464	}
1465
1466
1467	#ifdef IN_RING0
1468	/**
1469	* This is called during VM initialization to create an MMIO2 range.
1470	*
1471	* This does everything except setting the PGMRAMRANGE::pszDesc to a non-zero
1472	* value and preallocating the access handler for dirty bitmap tracking.
1473	*
1474	* The caller already knows which MMIO2 ID will be assigned to the registration
1475	* and how many chunks it requires, so there are no output fields in the request
1476	* structure.
1477	*
1478	* @returns VBox status code.
1479	* @param pGVM Pointer to the global VM structure.
1480	* @param pReq Where to get the parameters.
1481	* @thread EMT(0)
1482	*/
1483	VMMR0_INT_DECL(int) PGMR0PhysMmio2RegisterReq(PGVM pGVM, PPGMPHYSMMIO2REGISTERREQ pReq)
1484	{
1485	/*
1486	* Validate input (ASSUME pReq is a copy and can't be modified by ring-3
1487	* while we're here).
1488	*/
1489	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
1490	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x < %#zx\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
1491
1492	/** @todo better VM state guard, enmVMState is ring-3 writable. */
1493	VMSTATE const enmState = pGVM->enmVMState;
1494	AssertMsgReturn( enmState == VMSTATE_CREATING
1495	\|\| enmState == VMSTATE_LOADING /* pre 4.3.6 state loading needs to ignore a MMIO2 region in PCNet. */
1496	, ("enmState=%d\n", enmState), VERR_VM_INVALID_VM_STATE);
1497	VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
1498
1499	AssertReturn(pReq->cbGuestPage == GUEST_PAGE_SIZE, VERR_INCOMPATIBLE_CONFIG);
1500	AssertReturn(GUEST_PAGE_SIZE == HOST_PAGE_SIZE_DYNAMIC, VERR_INCOMPATIBLE_CONFIG);
1501
1502	AssertReturn(pReq->cGuestPages > 0, VERR_OUT_OF_RANGE);
1503	AssertReturn(pReq->cGuestPages <= PGM_MAX_PAGES_PER_MMIO2_REGION, VERR_OUT_OF_RANGE);
1504	AssertReturn(pReq->cGuestPages <= (MM_MMIO_64_MAX >> GUEST_PAGE_SHIFT), VERR_OUT_OF_RANGE);
1505
1506	AssertMsgReturn(!(pReq->fFlags & ~PGMPHYS_MMIO2_FLAGS_VALID_MASK), ("fFlags=%#x\n", pReq->fFlags), VERR_INVALID_FLAGS);
1507
1508	AssertMsgReturn( pReq->cChunks > 0
1509	&& pReq->cChunks < PGM_MAX_MMIO2_RANGES
1510	&& pReq->cChunks == pgmPhysMmio2CalcChunkCount((RTGCPHYS)pReq->cGuestPages << GUEST_PAGE_SHIFT, NULL),
1511	("cChunks=%#x cGuestPages=%#x\n", pReq->cChunks, pReq->cGuestPages),
1512	VERR_INVALID_PARAMETER);
1513
1514	AssertMsgReturn( pReq->idMmio2 != 0
1515	&& pReq->idMmio2 <= PGM_MAX_MMIO2_RANGES
1516	&& (unsigned)pReq->idMmio2 + pReq->cChunks - 1U <= PGM_MAX_MMIO2_RANGES,
1517	("idMmio2=%#x cChunks=%#x\n", pReq->idMmio2, pReq->cChunks),
1518	VERR_INVALID_PARAMETER);
1519
1520	for (uint32_t iChunk = 0, idx = pReq->idMmio2 - 1; iChunk < pReq->cChunks; iChunk++, idx++)
1521	{
1522	AssertReturn(pGVM->pgmr0.s.ahMmio2MapObjs[idx] == NIL_RTR0MEMOBJ, VERR_INVALID_STATE);
1523	AssertReturn(pGVM->pgmr0.s.ahMmio2MemObjs[idx] == NIL_RTR0MEMOBJ, VERR_INVALID_STATE);
1524	AssertReturn(pGVM->pgmr0.s.apMmio2RamRanges[idx] == NULL, VERR_INVALID_STATE);
1525	}
1526
1527	/*
1528	* Make sure we're owning the PGM lock (caller should be), recheck idMmio2
1529	* and call the worker function we share with ring-3.
1530	*/
1531	int rc = PGM_LOCK(pGVM);
1532	AssertRCReturn(rc, rc);
1533
1534	AssertReturnStmt(pGVM->pgm.s.cMmio2Ranges + 1U == pReq->idMmio2,
1535	PGM_UNLOCK(pGVM), VERR_INVALID_PARAMETER);
1536	AssertReturnStmt(pGVM->pgmr0.s.idRamRangeMax + 1U + pReq->cChunks <= RT_ELEMENTS(pGVM->pgmr0.s.apRamRanges),
1537	PGM_UNLOCK(pGVM), VERR_PGM_TOO_MANY_RAM_RANGES);
1538
1539	rc = pgmPhysMmio2RegisterWorker(pGVM, pReq->cGuestPages, pReq->idMmio2, pReq->cChunks,
1540	pReq->pDevIns, pReq->iSubDev, pReq->iRegion, pReq->fFlags);
1541
1542	PGM_UNLOCK(pGVM);
1543	return rc;
1544	}
1545	#endif /* IN_RING0 */
1546
1547
1548
1549	/**
1550	* Worker for PGMR3PhysMmio2Deregister & PGMR0PhysMmio2DeregisterReq.
1551	*/
1552	DECLHIDDEN(int) pgmPhysMmio2DeregisterWorker(PVMCC pVM, uint8_t idMmio2, uint8_t cChunks, PPDMDEVINSR3 pDevIns)
1553	{
1554	/*
1555	* The caller shall have made sure all this is true, but we check again
1556	* since we're paranoid.
1557	*/
1558	AssertReturn(idMmio2 > 0 && idMmio2 <= RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges), VERR_INTERNAL_ERROR_2);
1559	AssertReturn(cChunks >= 1, VERR_INTERNAL_ERROR_2);
1560	uint8_t const idxFirst = idMmio2 - 1U;
1561	AssertReturn(idxFirst + cChunks <= pVM->pgm.s.cMmio2Ranges, VERR_INTERNAL_ERROR_2);
1562	uint32_t cGuestPages = 0; /* (For accounting and calulating backing memory size) */
1563	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
1564	{
1565	AssertReturn(pVM->pgm.s.aMmio2Ranges[idx].pDevInsR3 == pDevIns, VERR_NOT_OWNER);
1566	AssertReturn(!(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_MAPPED), VERR_RESOURCE_BUSY);
1567	AssertReturn(pVM->pgm.s.aMmio2Ranges[idx].GCPhys == NIL_RTGCPHYS, VERR_INVALID_STATE);
1568	if (iChunk == 0)
1569	AssertReturn(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK, VERR_INVALID_PARAMETER);
1570	else
1571	AssertReturn(!(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK), VERR_INVALID_PARAMETER);
1572	if (iChunk + 1 == cChunks)
1573	AssertReturn(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK, VERR_INVALID_PARAMETER);
1574	else
1575	AssertReturn(!(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK), VERR_INVALID_PARAMETER);
1576	AssertReturn(pVM->pgm.s.aMmio2Ranges[idx].pPhysHandlerR3 == NIL_RTR3PTR, VERR_INVALID_STATE); /* caller shall free this */
1577
1578	#ifdef IN_RING0
1579	cGuestPages += pVM->pgmr0.s.acMmio2RangePages[idx];
1580	#else
1581	cGuestPages += pVM->pgm.s.aMmio2Ranges[idx].cbReal >> GUEST_PAGE_SHIFT;
1582	#endif
1583
1584	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apMmio2RamRanges[idx];
1585	AssertPtrReturn(pRamRange, VERR_INVALID_STATE);
1586	AssertReturn(pRamRange->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX, VERR_INVALID_STATE);
1587	AssertReturn(pRamRange->GCPhys == NIL_RTGCPHYS, VERR_INVALID_STATE);
1588	AssertReturn(pRamRange->GCPhysLast == NIL_RTGCPHYS, VERR_INVALID_STATE);
1589	}
1590
1591	/*
1592	* Remove everything except the backing memory first. We work the ranges
1593	* in reverse so that we can reduce the max RAM range ID when possible.
1594	*/
1595	#ifdef IN_RING3
1596	uint8_t * const pbMmio2Backing = pVM->pgm.s.aMmio2Ranges[idxFirst].pbR3;
1597	RTGCPHYS const cbMmio2Backing = RT_ALIGN_T((RTGCPHYS)cGuestPages << GUEST_PAGE_SHIFT, HOST_PAGE_SIZE_DYNAMIC, RTGCPHYS);
1598	#endif
1599
1600	int rc = VINF_SUCCESS;
1601	uint32_t iChunk = cChunks;
1602	while (iChunk-- > 0)
1603	{
1604	uint32_t const idx = idxFirst + iChunk;
1605	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apMmio2RamRanges[idx];
1606
1607	/* Zap the MMIO2 region data. */
1608	pVM->pgm.s.apMmio2RamRanges[idx] = NIL_RTR3PTR;
1609	#ifdef IN_RING0
1610	pVM->pgmr0.s.apMmio2RamRanges[idx] = NULL;
1611	pVM->pgmr0.s.acMmio2RangePages[idx] = 0;
1612	#endif
1613	pVM->pgm.s.aMmio2Ranges[idx].pDevInsR3 = NIL_RTR3PTR;
1614	pVM->pgm.s.aMmio2Ranges[idx].pbR3 = NIL_RTR3PTR;
1615	pVM->pgm.s.aMmio2Ranges[idx].fFlags = 0;
1616	pVM->pgm.s.aMmio2Ranges[idx].iSubDev = UINT8_MAX;
1617	pVM->pgm.s.aMmio2Ranges[idx].iRegion = UINT8_MAX;
1618	pVM->pgm.s.aMmio2Ranges[idx].idSavedState = UINT8_MAX;
1619	pVM->pgm.s.aMmio2Ranges[idx].idMmio2 = UINT8_MAX;
1620	pVM->pgm.s.aMmio2Ranges[idx].idRamRange = UINT16_MAX;
1621	pVM->pgm.s.aMmio2Ranges[idx].GCPhys = NIL_RTGCPHYS;
1622	pVM->pgm.s.aMmio2Ranges[idx].cbReal = 0;
1623	pVM->pgm.s.aMmio2Ranges[idx].pPhysHandlerR3 = NIL_RTR3PTR;
1624	pVM->pgm.s.aMmio2Ranges[idx].paLSPages = NIL_RTR3PTR;
1625
1626	/* Free the RAM range. */
1627	int rc2 = pgmPhysRamRangeFree(pVM, pRamRange);
1628	AssertLogRelMsgStmt(RT_SUCCESS(rc2), ("rc=%Rrc idx=%u chunk=%u/%u\n", rc, idx, iChunk + 1, cChunks),
1629	rc = RT_SUCCESS(rc) ? rc2 : rc);
1630	}
1631
1632	/*
1633	* Final removal frees up the backing memory.
1634	*/
1635	#ifdef IN_RING3
1636	int const rcBacking = SUPR3PageFree(pbMmio2Backing, cbMmio2Backing >> HOST_PAGE_SHIFT_DYNAMIC);
1637	AssertLogRelMsgStmt(RT_SUCCESS(rcBacking), ("rc=%Rrc %p LB %#zx\n", rcBacking, pbMmio2Backing, cbMmio2Backing),
1638	rc = RT_SUCCESS(rc) ? rcBacking : rc);
1639	#else
1640	int rcBacking = RTR0MemObjFree(pVM->pgmr0.s.ahMmio2MapObjs[idxFirst], true /fFreeMappings/);
1641	AssertLogRelMsgStmt(RT_SUCCESS(rcBacking),
1642	("rc=%Rrc ahMmio2MapObjs[%u]=%p\n", rcBacking, pVM->pgmr0.s.ahMmio2MapObjs[idxFirst], idxFirst),
1643	rc = RT_SUCCESS(rc) ? rcBacking : rc);
1644	if (RT_SUCCESS(rcBacking))
1645	{
1646	pVM->pgmr0.s.ahMmio2MapObjs[idxFirst] = NIL_RTR0MEMOBJ;
1647
1648	rcBacking = RTR0MemObjFree(pVM->pgmr0.s.ahMmio2MemObjs[idxFirst], true /fFreeMappings/);
1649	AssertLogRelMsgStmt(RT_SUCCESS(rcBacking),
1650	("rc=%Rrc ahMmio2MemObjs[%u]=%p\n", rcBacking, pVM->pgmr0.s.ahMmio2MemObjs[idxFirst], idxFirst),
1651	rc = RT_SUCCESS(rc) ? rcBacking : rc);
1652	if (RT_SUCCESS(rcBacking))
1653	pVM->pgmr0.s.ahMmio2MemObjs[idxFirst] = NIL_RTR0MEMOBJ;
1654	}
1655	#endif
1656
1657	/*
1658	* Decrease the MMIO2 count if these were the last ones.
1659	*/
1660	if (idxFirst + cChunks == pVM->pgm.s.cMmio2Ranges)
1661	pVM->pgm.s.cMmio2Ranges = idxFirst;
1662
1663	/*
1664	* Update page count stats.
1665	*/
1666	pVM->pgm.s.cAllPages -= cGuestPages;
1667	pVM->pgm.s.cPrivatePages -= cGuestPages;
1668
1669	return rc;
1670	}
1671
1672
1673	#ifdef IN_RING0
1674	/**
1675	* This is called during VM state loading to deregister an obsolete MMIO2 range.
1676	*
1677	* This does everything except TLB flushing and releasing the access handler.
1678	* The ranges must be unmapped and wihtout preallocated access handlers.
1679	*
1680	* @returns VBox status code.
1681	* @param pGVM Pointer to the global VM structure.
1682	* @param pReq Where to get the parameters.
1683	* @thread EMT(0)
1684	*/
1685	VMMR0_INT_DECL(int) PGMR0PhysMmio2DeregisterReq(PGVM pGVM, PPGMPHYSMMIO2DEREGISTERREQ pReq)
1686	{
1687	/*
1688	* Validate input (ASSUME pReq is a copy and can't be modified by ring-3
1689	* while we're here).
1690	*/
1691	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
1692	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x < %#zx\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
1693
1694	/** @todo better VM state guard, enmVMState is ring-3 writable. */
1695	/* Only LOADING, as this is special purpose for removing an unwanted PCNet MMIO2 region. */
1696	VMSTATE const enmState = pGVM->enmVMState;
1697	AssertMsgReturn(enmState == VMSTATE_LOADING, ("enmState=%d\n", enmState), VERR_VM_INVALID_VM_STATE);
1698	VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
1699
1700	AssertMsgReturn( pReq->cChunks > 0
1701	&& pReq->cChunks < PGM_MAX_MMIO2_RANGES,
1702	("idMmio2=%#x cChunks=%#x\n", pReq->idMmio2, pReq->cChunks),
1703	VERR_INVALID_PARAMETER);
1704
1705	AssertMsgReturn( pReq->idMmio2 != 0
1706	&& pReq->idMmio2 <= PGM_MAX_MMIO2_RANGES
1707	&& (unsigned)pReq->idMmio2 + pReq->cChunks - 1U <= PGM_MAX_MMIO2_RANGES,
1708	("idMmio2=%#x cChunks=%#x\n", pReq->idMmio2, pReq->cChunks),
1709	VERR_INVALID_PARAMETER);
1710
1711	/*
1712	* Validate that the requested range is for exactly one MMIO2 registration.
1713	*
1714	* This is safe to do w/o the lock because registration and deregistration
1715	* is restricted to EMT0, and we're on EMT0 so can't race ourselves.
1716	*/
1717
1718	/* Check that the first entry is valid and has a memory object for the backing memory. */
1719	uint32_t idx = pReq->idMmio2 - 1;
1720	AssertReturn(pGVM->pgmr0.s.apMmio2RamRanges[idx] != NULL, VERR_INVALID_STATE);
1721	AssertReturn(pGVM->pgmr0.s.ahMmio2MemObjs[idx] != NIL_RTR0MEMOBJ, VERR_INVALID_STATE);
1722
1723	/* Any additional regions must also have RAM ranges, but shall not have any backing memory. */
1724	idx++;
1725	for (uint32_t iChunk = 1; iChunk < pReq->cChunks; iChunk++, idx++)
1726	{
1727	AssertReturn(pGVM->pgmr0.s.apMmio2RamRanges[idx] != NULL, VERR_INVALID_STATE);
1728	AssertReturn(pGVM->pgmr0.s.ahMmio2MemObjs[idx] == NIL_RTR0MEMOBJ, VERR_INVALID_STATE);
1729	}
1730
1731	/* Check that the next entry is for a different region. */
1732	AssertReturn( idx >= RT_ELEMENTS(pGVM->pgmr0.s.apMmio2RamRanges)
1733	\|\| pGVM->pgmr0.s.apMmio2RamRanges[idx] == NULL
1734	\|\| pGVM->pgmr0.s.ahMmio2MemObjs[idx] != NIL_RTR0MEMOBJ,
1735	VERR_INVALID_PARAMETER);
1736
1737	/*
1738	* Make sure we're owning the PGM lock (caller should be) and call the
1739	* common worker code.
1740	*/
1741	int rc = PGM_LOCK(pGVM);
1742	AssertRCReturn(rc, rc);
1743
1744	rc = pgmPhysMmio2DeregisterWorker(pGVM, pReq->idMmio2, pReq->cChunks, pReq->pDevIns);
1745
1746	PGM_UNLOCK(pGVM);
1747	return rc;
1748	}
1749	#endif /* IN_RING0 */
1750
1751
1752
1753
1754	/*********************************************************************************************************************************
1755	* ROM *
1756	*********************************************************************************************************************************/
1757
1758
1759	/**
1760	* Common worker for pgmR3PhysRomRegisterLocked and
1761	* PGMR0PhysRomAllocateRangeReq.
1762	*/
1763	DECLHIDDEN(int) pgmPhysRomRangeAllocCommon(PVMCC pVM, uint32_t cPages, uint8_t idRomRange, uint32_t fFlags)
1764	{
1765	/*
1766	* Allocate the ROM range structure and map it into ring-3.
1767	*/
1768	size_t const cbRomRange = RT_ALIGN_Z(RT_UOFFSETOF_DYN(PGMROMRANGE, aPages[cPages]), HOST_PAGE_SIZE_DYNAMIC);
1769	#ifdef IN_RING0
1770	RTR0MEMOBJ hMemObj = NIL_RTR0MEMOBJ;
1771	int rc = RTR0MemObjAllocPage(&hMemObj, cbRomRange, false /fExecutable/);
1772	#else
1773	PPGMROMRANGE pRomRange;
1774	int rc = SUPR3PageAlloc(cbRomRange >> HOST_PAGE_SHIFT_DYNAMIC, 0 /fFlags/, (void **)&pRomRange);
1775	#endif
1776	if (RT_SUCCESS(rc))
1777	{
1778	/* Zero the memory and do basic range init before mapping it into userland. */
1779	#ifdef IN_RING0
1780	PPGMROMRANGE const pRomRange = (PPGMROMRANGE)RTR0MemObjAddress(hMemObj);
1781	if (!RTR0MemObjWasZeroInitialized(hMemObj))
1782	#endif
1783	RT_BZERO(pRomRange, cbRomRange);
1784
1785	pRomRange->GCPhys = NIL_RTGCPHYS;
1786	pRomRange->GCPhysLast = NIL_RTGCPHYS;
1787	pRomRange->cb = (RTGCPHYS)cPages << GUEST_PAGE_SHIFT;
1788	pRomRange->fFlags = fFlags;
1789	pRomRange->idSavedState = UINT8_MAX;
1790	pRomRange->idRamRange = UINT16_MAX;
1791	pRomRange->cbOriginal = 0;
1792	pRomRange->pvOriginal = NIL_RTR3PTR;
1793	pRomRange->pszDesc = NIL_RTR3PTR;
1794
1795	#ifdef IN_RING0
1796	/* Map it into userland. */
1797	RTR0MEMOBJ hMapObj = NIL_RTR0MEMOBJ;
1798	rc = RTR0MemObjMapUser(&hMapObj, hMemObj, (RTR3PTR)-1, 0 /uAlignment/,
1799	RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
1800	if (RT_SUCCESS(rc))
1801	#endif
1802	{
1803	/*
1804	* Grab the lock (unlikely to fail or block as caller typically owns it already).
1805	*/
1806	rc = PGM_LOCK(pVM);
1807	if (RT_SUCCESS(rc))
1808	{
1809	/*
1810	* Check that idRomRange is still free.
1811	*/
1812	if (idRomRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRomRanges))
1813	{
1814	#ifdef IN_RING0
1815	if (pVM->pgmr0.s.apRomRanges[idRomRange] == NULL)
1816	#endif
1817	{
1818	if ( pVM->pgm.s.apRomRanges[idRomRange] == NIL_RTR3PTR
1819	&& pVM->pgm.s.cRomRanges == idRomRange)
1820	{
1821	/*
1822	* Commit it.
1823	*/
1824	#ifdef IN_RING0
1825	pVM->pgmr0.s.apRomRanges[idRomRange] = pRomRange;
1826	pVM->pgmr0.s.acRomRangePages[idRomRange] = cPages;
1827	pVM->pgmr0.s.ahRomRangeMemObjs[idRomRange] = hMemObj;
1828	pVM->pgmr0.s.ahRomRangeMapObjs[idRomRange] = hMapObj;
1829	#endif
1830
1831	pVM->pgm.s.cRomRanges = idRomRange + 1;
1832	#ifdef IN_RING0
1833	pVM->pgm.s.apRomRanges[idRomRange] = RTR0MemObjAddressR3(hMapObj);
1834	#else
1835	pVM->pgm.s.apRomRanges[idRomRange] = pRomRange;
1836	#endif
1837
1838	PGM_UNLOCK(pVM);
1839	return VINF_SUCCESS;
1840	}
1841	}
1842
1843	/*
1844	* Bail out.
1845	*/
1846	rc = VERR_INTERNAL_ERROR_5;
1847	}
1848	else
1849	rc = VERR_PGM_TOO_MANY_ROM_RANGES;
1850	PGM_UNLOCK(pVM);
1851	}
1852	#ifdef IN_RING0
1853	RTR0MemObjFree(hMapObj, false /fFreeMappings/);
1854	#endif
1855	}
1856	#ifdef IN_RING0
1857	RTR0MemObjFree(hMemObj, true /fFreeMappings/);
1858	#else
1859	SUPR3PageFree(pRomRange, cbRomRange >> HOST_PAGE_SHIFT_DYNAMIC);
1860	#endif
1861	}
1862	return rc;
1863	}
1864
1865
1866	#ifdef IN_RING0
1867	/**
1868	* This is called during VM initialization to allocate a ROM range.
1869	*
1870	* The page array is zeroed, the rest is initialized as best we can based on the
1871	* information in @a pReq.
1872	*
1873	* @returns VBox status code.
1874	* @param pGVM Pointer to the global VM structure.
1875	* @param pReq Where to get the parameters and return the range ID.
1876	* @thread EMT(0)
1877	*/
1878	VMMR0_INT_DECL(int) PGMR0PhysRomAllocateRangeReq(PGVM pGVM, PPGMPHYSROMALLOCATERANGEREQ pReq)
1879	{
1880	/*
1881	* Validate input (ASSUME pReq is a copy and can't be modified by ring-3
1882	* while we're here).
1883	*/
1884	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
1885	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x < %#zx\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
1886
1887	AssertReturn(pReq->cbGuestPage == GUEST_PAGE_SIZE, VERR_INCOMPATIBLE_CONFIG);
1888
1889	AssertReturn(pReq->cGuestPages > 0, VERR_OUT_OF_RANGE);
1890	AssertReturn(pReq->cGuestPages <= PGM_MAX_PAGES_PER_ROM_RANGE, VERR_OUT_OF_RANGE);
1891
1892	AssertMsgReturn(!(pReq->fFlags & ~(uint32_t)PGMPHYS_ROM_FLAGS_VALID_MASK), ("fFlags=%#RX32\n", pReq->fFlags),
1893	VERR_INVALID_FLAGS);
1894
1895	AssertReturn(pReq->idRomRange < RT_ELEMENTS(pGVM->pgmr0.s.apRomRanges), VERR_OUT_OF_RANGE);
1896	AssertReturn(pReq->idRomRange == pGVM->pgm.s.cRomRanges, VERR_OUT_OF_RANGE);
1897
1898	/** @todo better VM state guard, enmVMState is ring-3 writable. */
1899	VMSTATE const enmState = pGVM->enmVMState;
1900	AssertMsgReturn(enmState == VMSTATE_CREATING, ("enmState=%d\n", enmState), VERR_VM_INVALID_VM_STATE);
1901	VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
1902
1903	/*
1904	* Call common worker.
1905	*/
1906	return pgmPhysRomRangeAllocCommon(pGVM, pReq->cGuestPages, pReq->idRomRange, pReq->fFlags);
1907	}
1908	#endif /* IN_RING0 */
1909
1910
1911	/*********************************************************************************************************************************
1912	* Other stuff
1913	*********************************************************************************************************************************/
1914
1915
1916
1917	/**
1918	* Checks if Address Gate 20 is enabled or not.
1919	*
1920	* @returns true if enabled.
1921	* @returns false if disabled.
1922	* @param pVCpu The cross context virtual CPU structure.
1923	*/
1924	VMMDECL(bool) PGMPhysIsA20Enabled(PVMCPU pVCpu)
1925	{
1926	/* Must check that pVCpu isn't NULL here because PDM device helper are a little lazy. */
1927	LogFlow(("PGMPhysIsA20Enabled %d\n", pVCpu && pVCpu->pgm.s.fA20Enabled));
1928	return pVCpu && pVCpu->pgm.s.fA20Enabled;
1929	}
1930
1931
1932	/**
1933	* Validates a GC physical address.
1934	*
1935	* @returns true if valid.
1936	* @returns false if invalid.
1937	* @param pVM The cross context VM structure.
1938	* @param GCPhys The physical address to validate.
1939	*/
1940	VMMDECL(bool) PGMPhysIsGCPhysValid(PVMCC pVM, RTGCPHYS GCPhys)
1941	{
1942	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
1943	return pPage != NULL;
1944	}
1945
1946
1947	/**
1948	* Checks if a GC physical address is a normal page,
1949	* i.e. not ROM, MMIO or reserved.
1950	*
1951	* @returns true if normal.
1952	* @returns false if invalid, ROM, MMIO or reserved page.
1953	* @param pVM The cross context VM structure.
1954	* @param GCPhys The physical address to check.
1955	*/
1956	VMMDECL(bool) PGMPhysIsGCPhysNormal(PVMCC pVM, RTGCPHYS GCPhys)
1957	{
1958	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
1959	return pPage
1960	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM;
1961	}
1962
1963
1964	/**
1965	* Converts a GC physical address to a HC physical address.
1966	*
1967	* @returns VINF_SUCCESS on success.
1968	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
1969	* page but has no physical backing.
1970	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
1971	* GC physical address.
1972	*
1973	* @param pVM The cross context VM structure.
1974	* @param GCPhys The GC physical address to convert.
1975	* @param pHCPhys Where to store the HC physical address on success.
1976	*/
1977	VMM_INT_DECL(int) PGMPhysGCPhys2HCPhys(PVMCC pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
1978	{
1979	PGM_LOCK_VOID(pVM);
1980	PPGMPAGE pPage;
1981	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1982	if (RT_SUCCESS(rc))
1983	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & GUEST_PAGE_OFFSET_MASK);
1984	PGM_UNLOCK(pVM);
1985	return rc;
1986	}
1987
1988
1989	/**
1990	* Invalidates all page mapping TLBs.
1991	*
1992	* @param pVM The cross context VM structure.
1993	* @param fInRendezvous Set if we're in a rendezvous.
1994	*/
1995	void pgmPhysInvalidatePageMapTLB(PVMCC pVM, bool fInRendezvous)
1996	{
1997	PGM_LOCK_VOID(pVM);
1998	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatPageMapTlbFlushes);
1999
2000	/* Clear the R3 & R0 TLBs completely. */
2001	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR0.aEntries); i++)
2002	{
2003	pVM->pgm.s.PhysTlbR0.aEntries[i].GCPhys = NIL_RTGCPHYS;
2004	pVM->pgm.s.PhysTlbR0.aEntries[i].pPage = 0;
2005	pVM->pgm.s.PhysTlbR0.aEntries[i].pv = 0;
2006	}
2007
2008	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR3.aEntries); i++)
2009	{
2010	pVM->pgm.s.PhysTlbR3.aEntries[i].GCPhys = NIL_RTGCPHYS;
2011	pVM->pgm.s.PhysTlbR3.aEntries[i].pPage = 0;
2012	pVM->pgm.s.PhysTlbR3.aEntries[i].pMap = 0;
2013	pVM->pgm.s.PhysTlbR3.aEntries[i].pv = 0;
2014	}
2015
2016	/* For the per VCPU lockless TLBs, we only invalid the GCPhys members so that
2017	anyone concurrently using the entry can safely continue to do so while any
2018	subsequent attempts to use it will fail. (Emulating a scenario where we
2019	lost the PGM lock race and the concurrent TLB user wont it.) */
2020	VMCC_FOR_EACH_VMCPU(pVM)
2021	{
2022	if (!fInRendezvous && pVCpu != VMMGetCpu(pVM))
2023	for (unsigned idx = 0; idx < RT_ELEMENTS(pVCpu->pgm.s.PhysTlb.aEntries); idx++)
2024	ASMAtomicWriteU64(&pVCpu->pgm.s.PhysTlb.aEntries[idx].GCPhys, NIL_RTGCPHYS);
2025	else
2026	for (unsigned idx = 0; idx < RT_ELEMENTS(pVCpu->pgm.s.PhysTlb.aEntries); idx++)
2027	pVCpu->pgm.s.PhysTlb.aEntries[idx].GCPhys = NIL_RTGCPHYS;
2028	}
2029	VMCC_FOR_EACH_VMCPU_END(pVM);
2030
2031	IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID, IEMTLBPHYSFLUSHREASON_MISC);
2032	PGM_UNLOCK(pVM);
2033	}
2034
2035
2036	/**
2037	* Invalidates a page mapping TLB entry
2038	*
2039	* @param pVM The cross context VM structure.
2040	* @param GCPhys GCPhys entry to flush
2041	*
2042	* @note Caller is responsible for calling IEMTlbInvalidateAllPhysicalAllCpus
2043	* when needed.
2044	*/
2045	void pgmPhysInvalidatePageMapTLBEntry(PVMCC pVM, RTGCPHYS GCPhys)
2046	{
2047	PGM_LOCK_ASSERT_OWNER(pVM);
2048
2049	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatPageMapTlbFlushEntry);
2050
2051	unsigned const idx = PGM_PAGER3MAPTLB_IDX(GCPhys);
2052
2053	pVM->pgm.s.PhysTlbR0.aEntries[idx].GCPhys = NIL_RTGCPHYS;
2054	pVM->pgm.s.PhysTlbR0.aEntries[idx].pPage = 0;
2055	pVM->pgm.s.PhysTlbR0.aEntries[idx].pv = 0;
2056
2057	pVM->pgm.s.PhysTlbR3.aEntries[idx].GCPhys = NIL_RTGCPHYS;
2058	pVM->pgm.s.PhysTlbR3.aEntries[idx].pPage = 0;
2059	pVM->pgm.s.PhysTlbR3.aEntries[idx].pMap = 0;
2060	pVM->pgm.s.PhysTlbR3.aEntries[idx].pv = 0;
2061
2062	/* For the per VCPU lockless TLBs, we only invalid the GCPhys member so that
2063	anyone concurrently using the entry can safely continue to do so while any
2064	subsequent attempts to use it will fail. (Emulating a scenario where we
2065	lost the PGM lock race and the concurrent TLB user wont it.) */
2066	VMCC_FOR_EACH_VMCPU(pVM)
2067	{
2068	ASMAtomicWriteU64(&pVCpu->pgm.s.PhysTlb.aEntries[idx].GCPhys, NIL_RTGCPHYS);
2069	}
2070	VMCC_FOR_EACH_VMCPU_END(pVM);
2071	}
2072
2073	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2074
2075	/**
2076	* Makes sure that there is at least one handy page ready for use.
2077	*
2078	* This will also take the appropriate actions when reaching water-marks.
2079	*
2080	* @returns VBox status code.
2081	* @retval VINF_SUCCESS on success.
2082	* @retval VERR_EM_NO_MEMORY if we're really out of memory.
2083	*
2084	* @param pVM The cross context VM structure.
2085	*
2086	* @remarks Must be called from within the PGM critical section. It may
2087	* nip back to ring-3/0 in some cases.
2088	*/
2089	static int pgmPhysEnsureHandyPage(PVMCC pVM)
2090	{
2091	AssertMsg(pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", pVM->pgm.s.cHandyPages));
2092
2093	/*
2094	* Do we need to do anything special?
2095	*/
2096	# ifdef IN_RING3
2097	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_R3_ALLOC))
2098	# else
2099	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_RZ_TO_R3))
2100	# endif
2101	{
2102	/*
2103	* Allocate pages only if we're out of them, or in ring-3, almost out.
2104	*/
2105	# ifdef IN_RING3
2106	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_R3_ALLOC)
2107	# else
2108	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_ALLOC)
2109	# endif
2110	{
2111	Log(("PGM: cHandyPages=%u out of %u -> allocate more; VM_FF_PGM_NO_MEMORY=%RTbool\n",
2112	pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages), VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY) ));
2113	# ifdef IN_RING3
2114	int rc = PGMR3PhysAllocateHandyPages(pVM);
2115	# else
2116	int rc = pgmR0PhysAllocateHandyPages(pVM, VMMGetCpuId(pVM), false /fRing3/);
2117	# endif
2118	if (RT_UNLIKELY(rc != VINF_SUCCESS))
2119	{
2120	if (RT_FAILURE(rc))
2121	return rc;
2122	AssertMsgReturn(rc == VINF_EM_NO_MEMORY, ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
2123	if (!pVM->pgm.s.cHandyPages)
2124	{
2125	LogRel(("PGM: no more handy pages!\n"));
2126	return VERR_EM_NO_MEMORY;
2127	}
2128	Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES));
2129	Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY));
2130	# ifndef IN_RING3
2131	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); /* paranoia */
2132	# endif
2133	}
2134	AssertMsgReturn( pVM->pgm.s.cHandyPages > 0
2135	&& pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages),
2136	("%u\n", pVM->pgm.s.cHandyPages),
2137	VERR_PGM_HANDY_PAGE_IPE);
2138	}
2139	else
2140	{
2141	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_SET_FF)
2142	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
2143	# ifndef IN_RING3
2144	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_TO_R3)
2145	{
2146	Log(("PGM: VM_FF_TO_R3 - cHandyPages=%u out of %u\n", pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
2147	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3);
2148	}
2149	# endif
2150	}
2151	}
2152
2153	return VINF_SUCCESS;
2154	}
2155
2156
2157	/**
2158	* Replace a zero or shared page with new page that we can write to.
2159	*
2160	* @returns The following VBox status codes.
2161	* @retval VINF_SUCCESS on success, pPage is modified.
2162	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
2163	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
2164	*
2165	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
2166	*
2167	* @param pVM The cross context VM structure.
2168	* @param pPage The physical page tracking structure. This will
2169	* be modified on success.
2170	* @param GCPhys The address of the page.
2171	*
2172	* @remarks Must be called from within the PGM critical section. It may
2173	* nip back to ring-3/0 in some cases.
2174	*
2175	* @remarks This function shouldn't really fail, however if it does
2176	* it probably means we've screwed up the size of handy pages and/or
2177	* the low-water mark. Or, that some device I/O is causing a lot of
2178	* pages to be allocated while while the host is in a low-memory
2179	* condition. This latter should be handled elsewhere and in a more
2180	* controlled manner, it's on the @bugref{3170} todo list...
2181	*/
2182	int pgmPhysAllocPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
2183	{
2184	LogFlow(("pgmPhysAllocPage: %R[pgmpage] %RGp\n", pPage, GCPhys));
2185
2186	/*
2187	* Prereqs.
2188	*/
2189	PGM_LOCK_ASSERT_OWNER(pVM);
2190	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
2191	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
2192
2193	# ifdef PGM_WITH_LARGE_PAGES
2194	/*
2195	* Try allocate a large page if applicable.
2196	*/
2197	if ( PGMIsUsingLargePages(pVM)
2198	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2199	&& !VM_IS_NEM_ENABLED(pVM)) /** @todo NEM: Implement large pages support. */
2200	{
2201	RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
2202	PPGMPAGE pBasePage;
2203
2204	int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pBasePage);
2205	AssertRCReturn(rc, rc); /* paranoia; can't happen. */
2206	if (PGM_PAGE_GET_PDE_TYPE(pBasePage) == PGM_PAGE_PDE_TYPE_DONTCARE)
2207	{
2208	rc = pgmPhysAllocLargePage(pVM, GCPhys);
2209	if (rc == VINF_SUCCESS)
2210	return rc;
2211	}
2212	/* Mark the base as type page table, so we don't check over and over again. */
2213	PGM_PAGE_SET_PDE_TYPE(pVM, pBasePage, PGM_PAGE_PDE_TYPE_PT);
2214
2215	/* fall back to 4KB pages. */
2216	}
2217	# endif
2218
2219	/*
2220	* Flush any shadow page table mappings of the page.
2221	* When VBOX_WITH_NEW_LAZY_PAGE_ALLOC isn't defined, there shouldn't be any.
2222	*/
2223	bool fFlushTLBs = false;
2224	int rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhys, pPage, true /fFlushTLBs/, &fFlushTLBs);
2225	AssertMsgReturn(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc), RT_FAILURE(rc) ? rc : VERR_IPE_UNEXPECTED_STATUS);
2226
2227	/*
2228	* Ensure that we've got a page handy, take it and use it.
2229	*/
2230	int rc2 = pgmPhysEnsureHandyPage(pVM);
2231	if (RT_FAILURE(rc2))
2232	{
2233	if (fFlushTLBs)
2234	PGM_INVL_ALL_VCPU_TLBS(pVM);
2235	Assert(rc2 == VERR_EM_NO_MEMORY);
2236	return rc2;
2237	}
2238	/* re-assert preconditions since pgmPhysEnsureHandyPage may do a context switch. */
2239	PGM_LOCK_ASSERT_OWNER(pVM);
2240	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
2241	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
2242
2243	uint32_t iHandyPage = --pVM->pgm.s.cHandyPages;
2244	AssertMsg(iHandyPage < RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", iHandyPage));
2245	Assert(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys != NIL_GMMPAGEDESC_PHYS);
2246	Assert(!(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
2247	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idPage != NIL_GMM_PAGEID);
2248	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID);
2249
2250	/*
2251	* There are one or two action to be taken the next time we allocate handy pages:
2252	* - Tell the GMM (global memory manager) what the page is being used for.
2253	* (Speeds up replacement operations - sharing and defragmenting.)
2254	* - If the current backing is shared, it must be freed.
2255	*/
2256	const RTHCPHYS HCPhys = pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys;
2257	pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys = GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
2258
2259	void const *pvSharedPage = NULL;
2260	if (!PGM_PAGE_IS_SHARED(pPage))
2261	{
2262	Log2(("PGM: Replaced zero page %RGp with %#x / %RHp\n", GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
2263	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatRZPageReplaceZero);
2264	pVM->pgm.s.cZeroPages--;
2265	}
2266	else
2267	{
2268	/* Mark this shared page for freeing/dereferencing. */
2269	pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage = PGM_PAGE_GET_PAGEID(pPage);
2270	Assert(PGM_PAGE_GET_PAGEID(pPage) != NIL_GMM_PAGEID);
2271
2272	Log(("PGM: Replaced shared page %#x at %RGp with %#x / %RHp\n", PGM_PAGE_GET_PAGEID(pPage),
2273	GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
2274	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageReplaceShared));
2275	pVM->pgm.s.cSharedPages--;
2276
2277	/* Grab the address of the page so we can make a copy later on. (safe) */
2278	rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvSharedPage);
2279	AssertRC(rc);
2280	}
2281
2282	/*
2283	* Do the PGMPAGE modifications.
2284	*/
2285	pVM->pgm.s.cPrivatePages++;
2286	PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
2287	PGM_PAGE_SET_PAGEID(pVM, pPage, pVM->pgm.s.aHandyPages[iHandyPage].idPage);
2288	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
2289	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PT);
2290	pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
2291	IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID,
2292	!pvSharedPage
2293	? IEMTLBPHYSFLUSHREASON_ALLOCATED : IEMTLBPHYSFLUSHREASON_ALLOCATED_FROM_SHARED);
2294
2295	/* Copy the shared page contents to the replacement page. */
2296	if (!pvSharedPage)
2297	{ /* likely */ }
2298	else
2299	{
2300	/* Get the virtual address of the new page. */
2301	PGMPAGEMAPLOCK PgMpLck;
2302	void *pvNewPage;
2303	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvNewPage, &PgMpLck); AssertRC(rc);
2304	if (RT_SUCCESS(rc))
2305	{
2306	memcpy(pvNewPage, pvSharedPage, GUEST_PAGE_SIZE); /** @todo todo write ASMMemCopyPage */
2307	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2308	}
2309	}
2310
2311	if ( fFlushTLBs
2312	&& rc != VINF_PGM_GCPHYS_ALIASED)
2313	PGM_INVL_ALL_VCPU_TLBS(pVM);
2314
2315	/*
2316	* Notify NEM about the mapping change for this page.
2317	*
2318	* Note! Shadow ROM pages are complicated as they can definitely be
2319	* allocated while not visible, so play safe.
2320	*/
2321	if (VM_IS_NEM_ENABLED(pVM))
2322	{
2323	PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
2324	if ( enmType != PGMPAGETYPE_ROM_SHADOW
2325	\|\| pgmPhysGetPage(pVM, GCPhys) == pPage)
2326	{
2327	uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
2328	rc2 = NEMHCNotifyPhysPageAllocated(pVM, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, HCPhys,
2329	pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
2330	if (RT_SUCCESS(rc))
2331	PGM_PAGE_SET_NEM_STATE(pPage, u2State);
2332	else
2333	rc = rc2;
2334	}
2335	}
2336
2337	return rc;
2338	}
2339
2340	# ifdef PGM_WITH_LARGE_PAGES
2341
2342	/**
2343	* Replace a 2 MB range of zero pages with new pages that we can write to.
2344	*
2345	* @returns The following VBox status codes.
2346	* @retval VINF_SUCCESS on success, pPage is modified.
2347	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
2348	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
2349	*
2350	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
2351	*
2352	* @param pVM The cross context VM structure.
2353	* @param GCPhys The address of the page.
2354	*
2355	* @remarks Must be called from within the PGM critical section. It may block
2356	* on GMM and host mutexes/locks, leaving HM context.
2357	*/
2358	int pgmPhysAllocLargePage(PVMCC pVM, RTGCPHYS GCPhys)
2359	{
2360	RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
2361	LogFlow(("pgmPhysAllocLargePage: %RGp base %RGp\n", GCPhys, GCPhysBase));
2362	Assert(!VM_IS_NEM_ENABLED(pVM)); /** @todo NEM: Large page support. */
2363
2364	/*
2365	* Check Prereqs.
2366	*/
2367	PGM_LOCK_ASSERT_OWNER(pVM);
2368	Assert(PGMIsUsingLargePages(pVM));
2369
2370	/*
2371	* All the pages must be unallocated RAM pages, i.e. mapping the ZERO page.
2372	*/
2373	PPGMPAGE pFirstPage;
2374	int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pFirstPage);
2375	if ( RT_SUCCESS(rc)
2376	&& PGM_PAGE_GET_TYPE(pFirstPage) == PGMPAGETYPE_RAM
2377	&& PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ZERO)
2378	{
2379	/*
2380	* Further they should have PDE type set to PGM_PAGE_PDE_TYPE_DONTCARE,
2381	* since they are unallocated.
2382	*/
2383	unsigned uPDEType = PGM_PAGE_GET_PDE_TYPE(pFirstPage);
2384	Assert(uPDEType != PGM_PAGE_PDE_TYPE_PDE);
2385	if (uPDEType == PGM_PAGE_PDE_TYPE_DONTCARE)
2386	{
2387	/*
2388	* Now, make sure all the other pages in the 2 MB is in the same state.
2389	*/
2390	GCPhys = GCPhysBase;
2391	unsigned cLeft = _2M / GUEST_PAGE_SIZE;
2392	while (cLeft-- > 0)
2393	{
2394	PPGMPAGE pSubPage = pgmPhysGetPage(pVM, GCPhys);
2395	if ( pSubPage
2396	&& PGM_PAGE_GET_TYPE(pSubPage) == PGMPAGETYPE_RAM /* Anything other than ram implies monitoring. */
2397	&& PGM_PAGE_GET_STATE(pSubPage) == PGM_PAGE_STATE_ZERO) /* Allocated, monitored or shared means we can't use a large page here */
2398	{
2399	Assert(PGM_PAGE_GET_PDE_TYPE(pSubPage) == PGM_PAGE_PDE_TYPE_DONTCARE);
2400	GCPhys += GUEST_PAGE_SIZE;
2401	}
2402	else
2403	{
2404	LogFlow(("pgmPhysAllocLargePage: Found page %RGp with wrong attributes (type=%d; state=%d); cancel check.\n",
2405	GCPhys, pSubPage ? PGM_PAGE_GET_TYPE(pSubPage) : -1, pSubPage ? PGM_PAGE_GET_STATE(pSubPage) : -1));
2406
2407	/* Failed. Mark as requiring a PT so we don't check the whole thing again in the future. */
2408	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRefused);
2409	PGM_PAGE_SET_PDE_TYPE(pVM, pFirstPage, PGM_PAGE_PDE_TYPE_PT);
2410	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
2411	}
2412	}
2413
2414	/*
2415	* Do the allocation.
2416	*/
2417	# ifdef IN_RING3
2418	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_PAGE, GCPhysBase, NULL);
2419	# elif defined(IN_RING0)
2420	rc = pgmR0PhysAllocateLargePage(pVM, VMMGetCpuId(pVM), GCPhysBase);
2421	# else
2422	# error "Port me"
2423	# endif
2424	if (RT_SUCCESS(rc))
2425	{
2426	Assert(PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ALLOCATED);
2427	pVM->pgm.s.cLargePages++;
2428	return VINF_SUCCESS;
2429	}
2430
2431	/* If we fail once, it most likely means the host's memory is too
2432	fragmented; don't bother trying again. */
2433	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
2434	return rc;
2435	}
2436	}
2437	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
2438	}
2439
2440
2441	/**
2442	* Recheck the entire 2 MB range to see if we can use it again as a large page.
2443	*
2444	* @returns The following VBox status codes.
2445	* @retval VINF_SUCCESS on success, the large page can be used again
2446	* @retval VERR_PGM_INVALID_LARGE_PAGE_RANGE if it can't be reused
2447	*
2448	* @param pVM The cross context VM structure.
2449	* @param GCPhys The address of the page.
2450	* @param pLargePage Page structure of the base page
2451	*/
2452	int pgmPhysRecheckLargePage(PVMCC pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage)
2453	{
2454	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRecheck);
2455
2456	Assert(!VM_IS_NEM_ENABLED(pVM)); /** @todo NEM: Large page support. */
2457
2458	AssertCompile(X86_PDE2M_PAE_PG_MASK == EPT_PDE2M_PG_MASK); /* Paranoia: Caller uses this for guest EPT tables as well. */
2459	GCPhys &= X86_PDE2M_PAE_PG_MASK;
2460
2461	/* Check the base page. */
2462	Assert(PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED);
2463	if ( PGM_PAGE_GET_STATE(pLargePage) != PGM_PAGE_STATE_ALLOCATED
2464	\|\| PGM_PAGE_GET_TYPE(pLargePage) != PGMPAGETYPE_RAM
2465	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
2466	{
2467	LogFlow(("pgmPhysRecheckLargePage: checks failed for base page %x %x %x\n", PGM_PAGE_GET_STATE(pLargePage), PGM_PAGE_GET_TYPE(pLargePage), PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage)));
2468	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
2469	}
2470
2471	STAM_PROFILE_START(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,IsValidLargePage), a);
2472	/* Check all remaining pages in the 2 MB range. */
2473	unsigned i;
2474	GCPhys += GUEST_PAGE_SIZE;
2475	for (i = 1; i < _2M / GUEST_PAGE_SIZE; i++)
2476	{
2477	PPGMPAGE pPage;
2478	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2479	AssertRCBreak(rc);
2480
2481	if ( PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2482	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
2483	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
2484	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
2485	{
2486	LogFlow(("pgmPhysRecheckLargePage: checks failed for page %d; %x %x %x\n", i, PGM_PAGE_GET_STATE(pPage), PGM_PAGE_GET_TYPE(pPage), PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)));
2487	break;
2488	}
2489
2490	GCPhys += GUEST_PAGE_SIZE;
2491	}
2492	STAM_PROFILE_STOP(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,IsValidLargePage), a);
2493
2494	if (i == _2M / GUEST_PAGE_SIZE)
2495	{
2496	PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE);
2497	pVM->pgm.s.cLargePagesDisabled--;
2498	Log(("pgmPhysRecheckLargePage: page %RGp can be reused!\n", GCPhys - _2M));
2499	return VINF_SUCCESS;
2500	}
2501
2502	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
2503	}
2504
2505	# endif /* PGM_WITH_LARGE_PAGES */
2506	#endif /* !VBOX_WITH_ONLY_PGM_NEM_MODE */
2507
2508
2509
2510	/**
2511	* Deal with a write monitored page.
2512	*
2513	* @param pVM The cross context VM structure.
2514	* @param pPage The physical page tracking structure.
2515	* @param GCPhys The guest physical address of the page.
2516	* PGMPhysReleasePageMappingLock() passes NIL_RTGCPHYS in a
2517	* very unlikely situation where it is okay that we let NEM
2518	* fix the page access in a lazy fasion.
2519	*
2520	* @remarks Called from within the PGM critical section.
2521	*/
2522	void pgmPhysPageMakeWriteMonitoredWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
2523	{
2524	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED);
2525	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
2526	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
2527	if (PGM_PAGE_IS_CODE_PAGE(pPage))
2528	{
2529	PGM_PAGE_CLEAR_CODE_PAGE(pVM, pPage);
2530	IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID, IEMTLBPHYSFLUSHREASON_MADE_WRITABLE);
2531	}
2532
2533	Assert(pVM->pgm.s.cMonitoredPages > 0);
2534	pVM->pgm.s.cMonitoredPages--;
2535	pVM->pgm.s.cWrittenToPages++;
2536
2537	#ifdef VBOX_WITH_NATIVE_NEM
2538	/*
2539	* Notify NEM about the protection change so we won't spin forever.
2540	*
2541	* Note! NEM need to be handle to lazily correct page protection as we cannot
2542	* really get it 100% right here it seems. The page pool does this too.
2543	*/
2544	if (VM_IS_NEM_ENABLED(pVM) && GCPhys != NIL_RTGCPHYS)
2545	{
2546	uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
2547	PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
2548	PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys);
2549	NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage),
2550	pRam ? PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhys) : NULL,
2551	pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
2552	PGM_PAGE_SET_NEM_STATE(pPage, u2State);
2553	}
2554	#else
2555	RT_NOREF(GCPhys);
2556	#endif
2557	}
2558
2559
2560	/**
2561	* Deal with pages that are not writable, i.e. not in the ALLOCATED state.
2562	*
2563	* @returns VBox strict status code.
2564	* @retval VINF_SUCCESS on success.
2565	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
2566	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2567	*
2568	* @param pVM The cross context VM structure.
2569	* @param pPage The physical page tracking structure.
2570	* @param GCPhys The address of the page.
2571	*
2572	* @remarks Called from within the PGM critical section.
2573	*/
2574	int pgmPhysPageMakeWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
2575	{
2576	PGM_LOCK_ASSERT_OWNER(pVM);
2577	switch (PGM_PAGE_GET_STATE(pPage))
2578	{
2579	case PGM_PAGE_STATE_WRITE_MONITORED:
2580	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
2581	RT_FALL_THRU();
2582	default: /* to shut up GCC */
2583	case PGM_PAGE_STATE_ALLOCATED:
2584	return VINF_SUCCESS;
2585
2586	/*
2587	* Zero pages can be dummy pages for MMIO or reserved memory,
2588	* so we need to check the flags before joining cause with
2589	* shared page replacement.
2590	*/
2591	case PGM_PAGE_STATE_ZERO:
2592	if (PGM_PAGE_IS_MMIO(pPage))
2593	return VERR_PGM_PHYS_PAGE_RESERVED;
2594	RT_FALL_THRU();
2595	case PGM_PAGE_STATE_SHARED:
2596	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2597	return pgmPhysAllocPage(pVM, pPage, GCPhys);
2598	#else
2599	AssertFailed(); /** @todo not sure if we make use of ZERO pages or not in NEM-mode, but I can't see how pgmPhysAllocPage would work. */
2600	return VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE;
2601	#endif
2602
2603	/* Not allowed to write to ballooned pages. */
2604	case PGM_PAGE_STATE_BALLOONED:
2605	return VERR_PGM_PHYS_PAGE_BALLOONED;
2606	}
2607	}
2608
2609	#if 0 /* unused */
2610	/**
2611	* Internal usage: Map the page specified by its GMM ID.
2612	*
2613	* This is similar to pgmPhysPageMap
2614	*
2615	* @returns VBox status code.
2616	*
2617	* @param pVM The cross context VM structure.
2618	* @param idPage The Page ID.
2619	* @param HCPhys The physical address (for SUPR0HCPhysToVirt).
2620	* @param ppv Where to store the mapping address.
2621	*
2622	* @remarks Called from within the PGM critical section. The mapping is only
2623	* valid while you are inside this section.
2624	*/
2625	int pgmPhysPageMapByPageID(PVMCC pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv)
2626	{
2627	/*
2628	* Validation.
2629	*/
2630	PGM_LOCK_ASSERT_OWNER(pVM);
2631	AssertReturn(HCPhys && !(HCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2632	const uint32_t idChunk = idPage >> GMM_CHUNKID_SHIFT;
2633	AssertReturn(idChunk != NIL_GMM_CHUNKID, VERR_INVALID_PARAMETER);
2634
2635	#ifdef IN_RING0
2636	# ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
2637	return SUPR0HCPhysToVirt(HCPhys & ~(RTHCPHYS)GUEST_PAGE_OFFSET_MASK, ppv);
2638	# else
2639	return GMMR0PageIdToVirt(pVM, idPage, ppv);
2640	# endif
2641
2642	#else
2643	/*
2644	* Find/make Chunk TLB entry for the mapping chunk.
2645	*/
2646	PPGMCHUNKR3MAP pMap;
2647	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
2648	if (pTlbe->idChunk == idChunk)
2649	{
2650	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbHits));
2651	pMap = pTlbe->pChunk;
2652	}
2653	else
2654	{
2655	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
2656
2657	/*
2658	* Find the chunk, map it if necessary.
2659	*/
2660	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
2661	if (pMap)
2662	pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
2663	else
2664	{
2665	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
2666	if (RT_FAILURE(rc))
2667	return rc;
2668	}
2669
2670	/*
2671	* Enter it into the Chunk TLB.
2672	*/
2673	pTlbe->idChunk = idChunk;
2674	pTlbe->pChunk = pMap;
2675	}
2676
2677	ppv = (uint8_t )pMap->pv + ((idPage & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT);
2678	return VINF_SUCCESS;
2679	#endif
2680	}
2681	#endif /* unused */
2682
2683	/**
2684	* Maps a page into the current virtual address space so it can be accessed.
2685	*
2686	* @returns VBox status code.
2687	* @retval VINF_SUCCESS on success.
2688	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2689	*
2690	* @param pVM The cross context VM structure.
2691	* @param pPage The physical page tracking structure.
2692	* @param GCPhys The address of the page.
2693	* @param ppMap Where to store the address of the mapping tracking structure.
2694	* @param ppv Where to store the mapping address of the page. The page
2695	* offset is masked off!
2696	*
2697	* @remarks Called from within the PGM critical section.
2698	*/
2699	static int pgmPhysPageMapCommon(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv)
2700	{
2701	PGM_LOCK_ASSERT_OWNER(pVM);
2702	NOREF(GCPhys);
2703
2704	/*
2705	* Special cases: MMIO2 and specially aliased MMIO pages.
2706	*/
2707	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2
2708	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO)
2709	{
2710	*ppMap = NULL;
2711
2712	/* Decode the page id to a page in a MMIO2 ram range. */
2713	uint8_t const idMmio2 = PGM_MMIO2_PAGEID_GET_MMIO2_ID(PGM_PAGE_GET_PAGEID(pPage));
2714	uint32_t const iPage = PGM_MMIO2_PAGEID_GET_IDX(PGM_PAGE_GET_PAGEID(pPage));
2715	AssertLogRelMsgReturn((uint8_t)(idMmio2 - 1U) < RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges),
2716	("idMmio2=%u size=%u type=%u GCPHys=%#RGp Id=%u State=%u", idMmio2,
2717	RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges), PGM_PAGE_GET_TYPE(pPage), GCPhys,
2718	pPage->s.idPage, pPage->s.uStateY),
2719	VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2720	PPGMREGMMIO2RANGE const pMmio2Range = &pVM->pgm.s.aMmio2Ranges[idMmio2 - 1];
2721	AssertLogRelReturn(pMmio2Range, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2722	AssertLogRelReturn(pMmio2Range->idMmio2 == idMmio2, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2723	#ifndef IN_RING0
2724	uint32_t const idRamRange = pMmio2Range->idRamRange;
2725	AssertLogRelReturn(idRamRange < RT_ELEMENTS(pVM->pgm.s.apRamRanges), VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2726	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
2727	AssertLogRelReturn(pRamRange, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2728	AssertLogRelReturn(iPage < (pRamRange->cb >> GUEST_PAGE_SHIFT), VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2729	*ppv = pMmio2Range->pbR3 + ((uintptr_t)iPage << GUEST_PAGE_SHIFT);
2730	return VINF_SUCCESS;
2731
2732	#else /* IN_RING0 */
2733	AssertLogRelReturn(iPage < pVM->pgmr0.s.acMmio2RangePages[idMmio2 - 1], VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2734	# ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
2735	return SUPR0HCPhysToVirt(PGM_PAGE_GET_HCPHYS(pPage), ppv);
2736	# else
2737	AssertPtr(pVM->pgmr0.s.apbMmio2Backing[idMmio2 - 1]);
2738	*ppv = pVM->pgmr0.s.apbMmio2Backing[idMmio2 - 1] + ((uintptr_t)iPage << GUEST_PAGE_SHIFT);
2739	return VINF_SUCCESS;
2740	# endif
2741	#endif
2742	}
2743
2744	#ifdef VBOX_WITH_PGM_NEM_MODE
2745	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2746	if (pVM->pgm.s.fNemMode)
2747	# endif
2748	{
2749	# ifdef IN_RING3
2750	/*
2751	* Find the corresponding RAM range and use that to locate the mapping address.
2752	*/
2753	/** @todo Use the page ID for some kind of indexing as we do with MMIO2 above. */
2754	PPGMRAMRANGE const pRam = pgmPhysGetRange(pVM, GCPhys);
2755	AssertLogRelMsgReturn(pRam, ("%RTGp\n", GCPhys), VERR_INTERNAL_ERROR_3);
2756	size_t const idxPage = (GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT;
2757	Assert(pPage == &pRam->aPages[idxPage]);
2758	*ppMap = NULL;
2759	ppv = (uint8_t )pRam->pbR3 + (idxPage << GUEST_PAGE_SHIFT);
2760	return VINF_SUCCESS;
2761	# else
2762	AssertFailedReturn(VERR_INTERNAL_ERROR_2);
2763	# endif
2764	}
2765	#endif /* VBOX_WITH_PGM_NEM_MODE */
2766	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2767
2768	const uint32_t idChunk = PGM_PAGE_GET_CHUNKID(pPage);
2769	if (idChunk == NIL_GMM_CHUNKID)
2770	{
2771	AssertMsgReturn(PGM_PAGE_GET_PAGEID(pPage) == NIL_GMM_PAGEID, ("pPage=%R[pgmpage]\n", pPage),
2772	VERR_PGM_PHYS_PAGE_MAP_IPE_1);
2773	if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
2774	{
2775	AssertMsgReturn(PGM_PAGE_IS_ZERO(pPage), ("pPage=%R[pgmpage]\n", pPage),
2776	VERR_PGM_PHYS_PAGE_MAP_IPE_3);
2777	AssertMsgReturn(PGM_PAGE_GET_HCPHYS(pPage)== pVM->pgm.s.HCPhysZeroPg, ("pPage=%R[pgmpage]\n", pPage),
2778	VERR_PGM_PHYS_PAGE_MAP_IPE_4);
2779	*ppv = pVM->pgm.s.abZeroPg;
2780	}
2781	else
2782	*ppv = pVM->pgm.s.abZeroPg;
2783	*ppMap = NULL;
2784	return VINF_SUCCESS;
2785	}
2786
2787	# if defined(IN_RING0) && defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
2788	/*
2789	* Just use the physical address.
2790	*/
2791	*ppMap = NULL;
2792	return SUPR0HCPhysToVirt(PGM_PAGE_GET_HCPHYS(pPage), ppv);
2793
2794	# elif defined(IN_RING0)
2795	/*
2796	* Go by page ID thru GMMR0.
2797	*/
2798	*ppMap = NULL;
2799	return GMMR0PageIdToVirt(pVM, PGM_PAGE_GET_PAGEID(pPage), ppv);
2800
2801	# else
2802	/*
2803	* Find/make Chunk TLB entry for the mapping chunk.
2804	*/
2805	PPGMCHUNKR3MAP pMap;
2806	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
2807	if (pTlbe->idChunk == idChunk)
2808	{
2809	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbHits));
2810	pMap = pTlbe->pChunk;
2811	AssertPtr(pMap->pv);
2812	}
2813	else
2814	{
2815	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
2816
2817	/*
2818	* Find the chunk, map it if necessary.
2819	*/
2820	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
2821	if (pMap)
2822	{
2823	AssertPtr(pMap->pv);
2824	pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
2825	}
2826	else
2827	{
2828	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
2829	if (RT_FAILURE(rc))
2830	return rc;
2831	AssertPtr(pMap->pv);
2832	}
2833
2834	/*
2835	* Enter it into the Chunk TLB.
2836	*/
2837	pTlbe->idChunk = idChunk;
2838	pTlbe->pChunk = pMap;
2839	}
2840
2841	ppv = (uint8_t )pMap->pv + (PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) << GUEST_PAGE_SHIFT);
2842	*ppMap = pMap;
2843	return VINF_SUCCESS;
2844	# endif /* !IN_RING0 */
2845	#endif /* !VBOX_WITH_ONLY_PGM_NEM_MODE */
2846	}
2847
2848
2849	/**
2850	* Combination of pgmPhysPageMakeWritable and pgmPhysPageMapWritable.
2851	*
2852	* This is typically used is paths where we cannot use the TLB methods (like ROM
2853	* pages) or where there is no point in using them since we won't get many hits.
2854	*
2855	* @returns VBox strict status code.
2856	* @retval VINF_SUCCESS on success.
2857	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
2858	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2859	*
2860	* @param pVM The cross context VM structure.
2861	* @param pPage The physical page tracking structure.
2862	* @param GCPhys The address of the page.
2863	* @param ppv Where to store the mapping address of the page. The page
2864	* offset is masked off!
2865	*
2866	* @remarks Called from within the PGM critical section. The mapping is only
2867	* valid while you are inside section.
2868	*/
2869	int pgmPhysPageMakeWritableAndMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
2870	{
2871	int rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2872	if (RT_SUCCESS(rc))
2873	{
2874	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* returned */, ("%Rrc\n", rc));
2875	PPGMPAGEMAP pMapIgnore;
2876	int rc2 = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
2877	if (RT_FAILURE(rc2)) /* preserve rc */
2878	rc = rc2;
2879	}
2880	return rc;
2881	}
2882
2883
2884	/**
2885	* Maps a page into the current virtual address space so it can be accessed for
2886	* both writing and reading.
2887	*
2888	* This is typically used is paths where we cannot use the TLB methods (like ROM
2889	* pages) or where there is no point in using them since we won't get many hits.
2890	*
2891	* @returns VBox status code.
2892	* @retval VINF_SUCCESS on success.
2893	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2894	*
2895	* @param pVM The cross context VM structure.
2896	* @param pPage The physical page tracking structure. Must be in the
2897	* allocated state.
2898	* @param GCPhys The address of the page.
2899	* @param ppv Where to store the mapping address of the page. The page
2900	* offset is masked off!
2901	*
2902	* @remarks Called from within the PGM critical section. The mapping is only
2903	* valid while you are inside section.
2904	*/
2905	int pgmPhysPageMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
2906	{
2907	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2908	PPGMPAGEMAP pMapIgnore;
2909	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
2910	}
2911
2912
2913	/**
2914	* Maps a page into the current virtual address space so it can be accessed for
2915	* reading.
2916	*
2917	* This is typically used is paths where we cannot use the TLB methods (like ROM
2918	* pages) or where there is no point in using them since we won't get many hits.
2919	*
2920	* @returns VBox status code.
2921	* @retval VINF_SUCCESS on success.
2922	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2923	*
2924	* @param pVM The cross context VM structure.
2925	* @param pPage The physical page tracking structure.
2926	* @param GCPhys The address of the page.
2927	* @param ppv Where to store the mapping address of the page. The page
2928	* offset is masked off!
2929	*
2930	* @remarks Called from within the PGM critical section. The mapping is only
2931	* valid while you are inside this section.
2932	*/
2933	int pgmPhysPageMapReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv)
2934	{
2935	PPGMPAGEMAP pMapIgnore;
2936	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, (void **)ppv);
2937	}
2938
2939
2940	/**
2941	* Load a guest page into the ring-3 physical TLB.
2942	*
2943	* @returns VBox status code.
2944	* @retval VINF_SUCCESS on success
2945	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
2946	* @param pVM The cross context VM structure.
2947	* @param GCPhys The guest physical address in question.
2948	*/
2949	int pgmPhysPageLoadIntoTlb(PVMCC pVM, RTGCPHYS GCPhys)
2950	{
2951	PGM_LOCK_ASSERT_OWNER(pVM);
2952
2953	/*
2954	* Find the ram range and page and hand it over to the with-page function.
2955	* 99.8% of requests are expected to be in the first range.
2956	*/
2957	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
2958	if (!pPage)
2959	{
2960	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageMapTlbMisses));
2961	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
2962	}
2963
2964	return pgmPhysPageLoadIntoTlbWithPage(pVM, pPage, GCPhys);
2965	}
2966
2967
2968	/**
2969	* Load a guest page into the ring-3 physical TLB.
2970	*
2971	* @returns VBox status code.
2972	* @retval VINF_SUCCESS on success
2973	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
2974	*
2975	* @param pVM The cross context VM structure.
2976	* @param pPage Pointer to the PGMPAGE structure corresponding to
2977	* GCPhys.
2978	* @param GCPhys The guest physical address in question.
2979	*/
2980	int pgmPhysPageLoadIntoTlbWithPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
2981	{
2982	PGM_LOCK_ASSERT_OWNER(pVM);
2983	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageMapTlbMisses));
2984
2985	/*
2986	* Map the page.
2987	* Make a special case for the zero page as it is kind of special.
2988	*/
2989	PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTX_SUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
2990	if ( !PGM_PAGE_IS_ZERO(pPage)
2991	&& !PGM_PAGE_IS_BALLOONED(pPage))
2992	{
2993	void *pv;
2994	PPGMPAGEMAP pMap;
2995	int rc = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMap, &pv);
2996	if (RT_FAILURE(rc))
2997	return rc;
2998	#ifndef IN_RING0
2999	pTlbe->pMap = pMap;
3000	#endif
3001	pTlbe->pv = pv;
3002	Assert(!((uintptr_t)pTlbe->pv & GUEST_PAGE_OFFSET_MASK));
3003	}
3004	else
3005	{
3006	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
3007	AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == pVM->pgm.s.HCPhysZeroPg, ("%RGp/%R[pgmpage]\n", GCPhys, pPage));
3008	#ifndef IN_RING0
3009	pTlbe->pMap = NULL;
3010	#endif
3011	pTlbe->pv = pVM->pgm.s.abZeroPg;
3012	#else
3013	/*
3014	* Should not ever be used, as we don't implement zero pages for NEM mode currently and
3015	* MMIO accesses are not handled through the TLB.
3016	*/
3017	pTlbe->pv = (void )&g_abRTZero64K[0]; / Maximum granule size on ARM. */
3018	#endif
3019	}
3020	#ifdef PGM_WITH_PHYS_TLB
3021	if ( PGM_PAGE_GET_TYPE(pPage) < PGMPAGETYPE_ROM_SHADOW
3022	\|\| PGM_PAGE_GET_TYPE(pPage) > PGMPAGETYPE_ROM)
3023	pTlbe->GCPhys = GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
3024	else
3025	pTlbe->GCPhys = NIL_RTGCPHYS; /* ROM: Problematic because of the two pages. :-/ */
3026	#else
3027	pTlbe->GCPhys = NIL_RTGCPHYS;
3028	#endif
3029	pTlbe->pPage = pPage;
3030	return VINF_SUCCESS;
3031	}
3032
3033
3034	#ifdef IN_RING3 /** @todo Need ensure a ring-0 version gets invalidated safely */
3035	/**
3036	* Load a guest page into the lockless ring-3 physical TLB for the calling EMT.
3037	*
3038	* @returns VBox status code.
3039	* @retval VINF_SUCCESS on success
3040	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3041	*
3042	* @param pVCpu The cross context virtual CPU structure.
3043	* @param pPage Pointer to the PGMPAGE structure corresponding to
3044	* GCPhys.
3045	* @param GCPhys The guest physical address in question.
3046	*/
3047	DECLHIDDEN(int) pgmPhysPageLoadIntoLocklessTlbWithPage(PVMCPUCC pVCpu, PPGMPAGE pPage, RTGCPHYS GCPhys)
3048	{
3049	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageMapTlbMisses));
3050	PPGMPAGEMAPTLBE const pLocklessTlbe = &pVCpu->pgm.s.PhysTlb.aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
3051	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3052
3053	PGM_LOCK_VOID(pVM);
3054
3055	PPGMPAGEMAPTLBE pSharedTlbe;
3056	int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pSharedTlbe);
3057	if (RT_SUCCESS(rc))
3058	pLocklessTlbe = pSharedTlbe;
3059
3060	PGM_UNLOCK(pVM);
3061	return rc;
3062	}
3063	#endif /* IN_RING3 */
3064
3065
3066	/**
3067	* Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
3068	* own the PGM lock and therefore not need to lock the mapped page.
3069	*
3070	* @returns VBox status code.
3071	* @retval VINF_SUCCESS on success.
3072	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3073	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3074	*
3075	* @param pVM The cross context VM structure.
3076	* @param GCPhys The guest physical address of the page that should be mapped.
3077	* @param pPage Pointer to the PGMPAGE structure for the page.
3078	* @param ppv Where to store the address corresponding to GCPhys.
3079	*
3080	* @internal
3081	* @deprecated Use pgmPhysGCPhys2CCPtrInternalEx.
3082	*/
3083	int pgmPhysGCPhys2CCPtrInternalDepr(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
3084	{
3085	int rc;
3086	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
3087	PGM_LOCK_ASSERT_OWNER(pVM);
3088	pVM->pgm.s.cDeprecatedPageLocks++;
3089
3090	/*
3091	* Make sure the page is writable.
3092	*/
3093	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
3094	{
3095	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3096	if (RT_FAILURE(rc))
3097	return rc;
3098	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
3099	}
3100	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0 \|\| PGM_IS_IN_NEM_MODE(pVM));
3101
3102	/*
3103	* Get the mapping address.
3104	*/
3105	PPGMPAGEMAPTLBE pTlbe;
3106	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
3107	if (RT_FAILURE(rc))
3108	return rc;
3109	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3110	return VINF_SUCCESS;
3111	}
3112
3113
3114	/**
3115	* Locks a page mapping for writing.
3116	*
3117	* @param pVM The cross context VM structure.
3118	* @param pPage The page.
3119	* @param pTlbe The mapping TLB entry for the page.
3120	* @param pLock The lock structure (output).
3121	*/
3122	DECLINLINE(void) pgmPhysPageMapLockForWriting(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
3123	{
3124	# ifndef IN_RING0
3125	PPGMPAGEMAP pMap = pTlbe->pMap;
3126	if (pMap)
3127	pMap->cRefs++;
3128	# else
3129	RT_NOREF(pTlbe);
3130	# endif
3131
3132	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
3133	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
3134	{
3135	if (cLocks == 0)
3136	pVM->pgm.s.cWriteLockedPages++;
3137	PGM_PAGE_INC_WRITE_LOCKS(pPage);
3138	}
3139	else if (cLocks != PGM_PAGE_MAX_LOCKS)
3140	{
3141	PGM_PAGE_INC_WRITE_LOCKS(pPage);
3142	AssertMsgFailed(("%R[pgmpage] is entering permanent write locked state!\n", pPage));
3143	# ifndef IN_RING0
3144	if (pMap)
3145	pMap->cRefs++; /* Extra ref to prevent it from going away. */
3146	# endif
3147	}
3148
3149	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
3150	# ifndef IN_RING0
3151	pLock->pvMap = pMap;
3152	# else
3153	pLock->pvMap = NULL;
3154	# endif
3155	}
3156
3157	/**
3158	* Locks a page mapping for reading.
3159	*
3160	* @param pVM The cross context VM structure.
3161	* @param pPage The page.
3162	* @param pTlbe The mapping TLB entry for the page.
3163	* @param pLock The lock structure (output).
3164	*/
3165	DECLINLINE(void) pgmPhysPageMapLockForReading(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
3166	{
3167	# ifndef IN_RING0
3168	PPGMPAGEMAP pMap = pTlbe->pMap;
3169	if (pMap)
3170	pMap->cRefs++;
3171	# else
3172	RT_NOREF(pTlbe);
3173	# endif
3174
3175	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
3176	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
3177	{
3178	if (cLocks == 0)
3179	pVM->pgm.s.cReadLockedPages++;
3180	PGM_PAGE_INC_READ_LOCKS(pPage);
3181	}
3182	else if (cLocks != PGM_PAGE_MAX_LOCKS)
3183	{
3184	PGM_PAGE_INC_READ_LOCKS(pPage);
3185	AssertMsgFailed(("%R[pgmpage] is entering permanent read locked state!\n", pPage));
3186	# ifndef IN_RING0
3187	if (pMap)
3188	pMap->cRefs++; /* Extra ref to prevent it from going away. */
3189	# endif
3190	}
3191
3192	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
3193	# ifndef IN_RING0
3194	pLock->pvMap = pMap;
3195	# else
3196	pLock->pvMap = NULL;
3197	# endif
3198	}
3199
3200
3201	/**
3202	* Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
3203	* own the PGM lock and have access to the page structure.
3204	*
3205	* @returns VBox status code.
3206	* @retval VINF_SUCCESS on success.
3207	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3208	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3209	*
3210	* @param pVM The cross context VM structure.
3211	* @param GCPhys The guest physical address of the page that should be mapped.
3212	* @param pPage Pointer to the PGMPAGE structure for the page.
3213	* @param ppv Where to store the address corresponding to GCPhys.
3214	* @param pLock Where to store the lock information that
3215	* pgmPhysReleaseInternalPageMappingLock needs.
3216	*
3217	* @internal
3218	*/
3219	int pgmPhysGCPhys2CCPtrInternal(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
3220	{
3221	int rc;
3222	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
3223	PGM_LOCK_ASSERT_OWNER(pVM);
3224
3225	/*
3226	* Make sure the page is writable.
3227	*/
3228	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
3229	{
3230	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3231	if (RT_FAILURE(rc))
3232	return rc;
3233	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
3234	}
3235	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0 \|\| PGM_IS_IN_NEM_MODE(pVM));
3236
3237	/*
3238	* Do the job.
3239	*/
3240	PPGMPAGEMAPTLBE pTlbe;
3241	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
3242	if (RT_FAILURE(rc))
3243	return rc;
3244	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
3245	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3246	return VINF_SUCCESS;
3247	}
3248
3249
3250	/**
3251	* Internal version of PGMPhysGCPhys2CCPtrReadOnly that expects the caller to
3252	* own the PGM lock and have access to the page structure.
3253	*
3254	* @returns VBox status code.
3255	* @retval VINF_SUCCESS on success.
3256	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3257	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3258	*
3259	* @param pVM The cross context VM structure.
3260	* @param GCPhys The guest physical address of the page that should be mapped.
3261	* @param pPage Pointer to the PGMPAGE structure for the page.
3262	* @param ppv Where to store the address corresponding to GCPhys.
3263	* @param pLock Where to store the lock information that
3264	* pgmPhysReleaseInternalPageMappingLock needs.
3265	*
3266	* @internal
3267	*/
3268	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv, PPGMPAGEMAPLOCK pLock)
3269	{
3270	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
3271	PGM_LOCK_ASSERT_OWNER(pVM);
3272	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0 \|\| PGM_IS_IN_NEM_MODE(pVM));
3273
3274	/*
3275	* Do the job.
3276	*/
3277	PPGMPAGEMAPTLBE pTlbe;
3278	int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
3279	if (RT_FAILURE(rc))
3280	return rc;
3281	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
3282	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3283	return VINF_SUCCESS;
3284	}
3285
3286
3287	/**
3288	* Requests the mapping of a guest page into the current context.
3289	*
3290	* This API should only be used for very short term, as it will consume scarse
3291	* resources (R0 and GC) in the mapping cache. When you're done with the page,
3292	* call PGMPhysReleasePageMappingLock() ASAP to release it.
3293	*
3294	* This API will assume your intention is to write to the page, and will
3295	* therefore replace shared and zero pages. If you do not intend to modify
3296	* the page, use the PGMPhysGCPhys2CCPtrReadOnly() API.
3297	*
3298	* @returns VBox status code.
3299	* @retval VINF_SUCCESS on success.
3300	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3301	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3302	*
3303	* @param pVM The cross context VM structure.
3304	* @param GCPhys The guest physical address of the page that should be
3305	* mapped.
3306	* @param ppv Where to store the address corresponding to GCPhys.
3307	* @param pLock Where to store the lock information that
3308	* PGMPhysReleasePageMappingLock needs.
3309	*
3310	* @remarks The caller is responsible for dealing with access handlers.
3311	* @todo Add an informational return code for pages with access handlers?
3312	*
3313	* @remark Avoid calling this API from within critical sections (other than
3314	* the PGM one) because of the deadlock risk. External threads may
3315	* need to delegate jobs to the EMTs.
3316	* @remarks Only one page is mapped! Make no assumption about what's after or
3317	* before the returned page!
3318	* @thread Any thread.
3319	*/
3320	VMM_INT_DECL(int) PGMPhysGCPhys2CCPtr(PVMCC pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
3321	{
3322	int rc = PGM_LOCK(pVM);
3323	AssertRCReturn(rc, rc);
3324
3325	/*
3326	* Query the Physical TLB entry for the page (may fail).
3327	*/
3328	PPGMPAGEMAPTLBE pTlbe;
3329	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
3330	if (RT_SUCCESS(rc))
3331	{
3332	/*
3333	* If the page is shared, the zero page, or being write monitored
3334	* it must be converted to a page that's writable if possible.
3335	*/
3336	PPGMPAGE pPage = pTlbe->pPage;
3337	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
3338	{
3339	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3340	if (RT_SUCCESS(rc))
3341	{
3342	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
3343	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
3344	}
3345	}
3346	if (RT_SUCCESS(rc))
3347	{
3348	/*
3349	* Now, just perform the locking and calculate the return address.
3350	*/
3351	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
3352	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3353	}
3354	}
3355
3356	PGM_UNLOCK(pVM);
3357	return rc;
3358	}
3359
3360
3361	/**
3362	* Requests the mapping of a guest page into the current context.
3363	*
3364	* This API should only be used for very short term, as it will consume scarse
3365	* resources (R0 and GC) in the mapping cache. When you're done with the page,
3366	* call PGMPhysReleasePageMappingLock() ASAP to release it.
3367	*
3368	* @returns VBox status code.
3369	* @retval VINF_SUCCESS on success.
3370	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3371	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3372	*
3373	* @param pVM The cross context VM structure.
3374	* @param GCPhys The guest physical address of the page that should be
3375	* mapped.
3376	* @param ppv Where to store the address corresponding to GCPhys.
3377	* @param pLock Where to store the lock information that
3378	* PGMPhysReleasePageMappingLock needs.
3379	*
3380	* @remarks The caller is responsible for dealing with access handlers.
3381	* @todo Add an informational return code for pages with access handlers?
3382	*
3383	* @remarks Avoid calling this API from within critical sections (other than
3384	* the PGM one) because of the deadlock risk.
3385	* @remarks Only one page is mapped! Make no assumption about what's after or
3386	* before the returned page!
3387	* @thread Any thread.
3388	*/
3389	VMM_INT_DECL(int) PGMPhysGCPhys2CCPtrReadOnly(PVMCC pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
3390	{
3391	int rc = PGM_LOCK(pVM);
3392	AssertRCReturn(rc, rc);
3393
3394	/*
3395	* Query the Physical TLB entry for the page (may fail).
3396	*/
3397	PPGMPAGEMAPTLBE pTlbe;
3398	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
3399	if (RT_SUCCESS(rc))
3400	{
3401	/* MMIO pages doesn't have any readable backing. */
3402	PPGMPAGE pPage = pTlbe->pPage;
3403	if (RT_UNLIKELY(PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)))
3404	rc = VERR_PGM_PHYS_PAGE_RESERVED;
3405	else
3406	{
3407	/*
3408	* Now, just perform the locking and calculate the return address.
3409	*/
3410	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
3411	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3412	}
3413	}
3414
3415	PGM_UNLOCK(pVM);
3416	return rc;
3417	}
3418
3419
3420	/**
3421	* Requests the mapping of a guest page given by virtual address into the current context.
3422	*
3423	* This API should only be used for very short term, as it will consume
3424	* scarse resources (R0 and GC) in the mapping cache. When you're done
3425	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
3426	*
3427	* This API will assume your intention is to write to the page, and will
3428	* therefore replace shared and zero pages. If you do not intend to modify
3429	* the page, use the PGMPhysGCPtr2CCPtrReadOnly() API.
3430	*
3431	* @returns VBox status code.
3432	* @retval VINF_SUCCESS on success.
3433	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
3434	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
3435	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3436	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3437	*
3438	* @param pVCpu The cross context virtual CPU structure.
3439	* @param GCPtr The guest physical address of the page that should be
3440	* mapped.
3441	* @param ppv Where to store the address corresponding to GCPhys.
3442	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
3443	*
3444	* @remark Avoid calling this API from within critical sections (other than
3445	* the PGM one) because of the deadlock risk.
3446	* @thread EMT
3447	*/
3448	VMM_INT_DECL(int) PGMPhysGCPtr2CCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, void **ppv, PPGMPAGEMAPLOCK pLock)
3449	{
3450	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
3451	RTGCPHYS GCPhys;
3452	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
3453	if (RT_SUCCESS(rc))
3454	rc = PGMPhysGCPhys2CCPtr(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
3455	return rc;
3456	}
3457
3458
3459	/**
3460	* Requests the mapping of a guest page given by virtual address into the current context.
3461	*
3462	* This API should only be used for very short term, as it will consume
3463	* scarse resources (R0 and GC) in the mapping cache. When you're done
3464	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
3465	*
3466	* @returns VBox status code.
3467	* @retval VINF_SUCCESS on success.
3468	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
3469	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
3470	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3471	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3472	*
3473	* @param pVCpu The cross context virtual CPU structure.
3474	* @param GCPtr The guest physical address of the page that should be
3475	* mapped.
3476	* @param ppv Where to store the address corresponding to GCPtr.
3477	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
3478	*
3479	* @remark Avoid calling this API from within critical sections (other than
3480	* the PGM one) because of the deadlock risk.
3481	* @thread EMT(pVCpu)
3482	*/
3483	VMM_INT_DECL(int) PGMPhysGCPtr2CCPtrReadOnly(PVMCPUCC pVCpu, RTGCPTR GCPtr, void const **ppv, PPGMPAGEMAPLOCK pLock)
3484	{
3485	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
3486	RTGCPHYS GCPhys;
3487	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
3488	if (RT_SUCCESS(rc))
3489	rc = PGMPhysGCPhys2CCPtrReadOnly(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
3490	return rc;
3491	}
3492
3493
3494	/**
3495	* Release the mapping of a guest page.
3496	*
3497	* This is the counter part of PGMPhysGCPhys2CCPtr, PGMPhysGCPhys2CCPtrReadOnly
3498	* PGMPhysGCPtr2CCPtr and PGMPhysGCPtr2CCPtrReadOnly.
3499	*
3500	* @param pVM The cross context VM structure.
3501	* @param pLock The lock structure initialized by the mapping function.
3502	*/
3503	VMMDECL(void) PGMPhysReleasePageMappingLock(PVMCC pVM, PPGMPAGEMAPLOCK pLock)
3504	{
3505	# ifndef IN_RING0
3506	PPGMPAGEMAP pMap = (PPGMPAGEMAP)pLock->pvMap;
3507	# endif
3508	PPGMPAGE pPage = (PPGMPAGE)(pLock->uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
3509	bool fWriteLock = (pLock->uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
3510
3511	pLock->uPageAndType = 0;
3512	pLock->pvMap = NULL;
3513
3514	PGM_LOCK_VOID(pVM);
3515	if (fWriteLock)
3516	{
3517	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
3518	Assert(cLocks > 0);
3519	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
3520	{
3521	if (cLocks == 1)
3522	{
3523	Assert(pVM->pgm.s.cWriteLockedPages > 0);
3524	pVM->pgm.s.cWriteLockedPages--;
3525	}
3526	PGM_PAGE_DEC_WRITE_LOCKS(pPage);
3527	}
3528
3529	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED)
3530	{ /* probably extremely likely */ }
3531	else
3532	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, NIL_RTGCPHYS);
3533	}
3534	else
3535	{
3536	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
3537	Assert(cLocks > 0);
3538	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
3539	{
3540	if (cLocks == 1)
3541	{
3542	Assert(pVM->pgm.s.cReadLockedPages > 0);
3543	pVM->pgm.s.cReadLockedPages--;
3544	}
3545	PGM_PAGE_DEC_READ_LOCKS(pPage);
3546	}
3547	}
3548
3549	# ifndef IN_RING0
3550	if (pMap)
3551	{
3552	Assert(pMap->cRefs >= 1);
3553	pMap->cRefs--;
3554	}
3555	# endif
3556	PGM_UNLOCK(pVM);
3557	}
3558
3559
3560	#ifdef IN_RING3
3561	/**
3562	* Release the mapping of multiple guest pages.
3563	*
3564	* This is the counter part to PGMR3PhysBulkGCPhys2CCPtrExternal() and
3565	* PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal().
3566	*
3567	* @param pVM The cross context VM structure.
3568	* @param cPages Number of pages to unlock.
3569	* @param paLocks Array of locks lock structure initialized by the mapping
3570	* function.
3571	*/
3572	VMMDECL(void) PGMPhysBulkReleasePageMappingLocks(PVMCC pVM, uint32_t cPages, PPGMPAGEMAPLOCK paLocks)
3573	{
3574	Assert(cPages > 0);
3575	bool const fWriteLock = (paLocks[0].uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
3576	#ifdef VBOX_STRICT
3577	for (uint32_t i = 1; i < cPages; i++)
3578	{
3579	Assert(fWriteLock == ((paLocks[i].uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE));
3580	AssertPtr(paLocks[i].uPageAndType);
3581	}
3582	#endif
3583
3584	PGM_LOCK_VOID(pVM);
3585	if (fWriteLock)
3586	{
3587	/*
3588	* Write locks:
3589	*/
3590	for (uint32_t i = 0; i < cPages; i++)
3591	{
3592	PPGMPAGE pPage = (PPGMPAGE)(paLocks[i].uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
3593	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
3594	Assert(cLocks > 0);
3595	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
3596	{
3597	if (cLocks == 1)
3598	{
3599	Assert(pVM->pgm.s.cWriteLockedPages > 0);
3600	pVM->pgm.s.cWriteLockedPages--;
3601	}
3602	PGM_PAGE_DEC_WRITE_LOCKS(pPage);
3603	}
3604
3605	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED)
3606	{ /* probably extremely likely */ }
3607	else
3608	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, NIL_RTGCPHYS);
3609
3610	PPGMPAGEMAP pMap = (PPGMPAGEMAP)paLocks[i].pvMap;
3611	if (pMap)
3612	{
3613	Assert(pMap->cRefs >= 1);
3614	pMap->cRefs--;
3615	}
3616
3617	/* Yield the lock: */
3618	if ((i & 1023) == 1023 && i + 1 < cPages)
3619	{
3620	PGM_UNLOCK(pVM);
3621	PGM_LOCK_VOID(pVM);
3622	}
3623	}
3624	}
3625	else
3626	{
3627	/*
3628	* Read locks:
3629	*/
3630	for (uint32_t i = 0; i < cPages; i++)
3631	{
3632	PPGMPAGE pPage = (PPGMPAGE)(paLocks[i].uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
3633	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
3634	Assert(cLocks > 0);
3635	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
3636	{
3637	if (cLocks == 1)
3638	{
3639	Assert(pVM->pgm.s.cReadLockedPages > 0);
3640	pVM->pgm.s.cReadLockedPages--;
3641	}
3642	PGM_PAGE_DEC_READ_LOCKS(pPage);
3643	}
3644
3645	PPGMPAGEMAP pMap = (PPGMPAGEMAP)paLocks[i].pvMap;
3646	if (pMap)
3647	{
3648	Assert(pMap->cRefs >= 1);
3649	pMap->cRefs--;
3650	}
3651
3652	/* Yield the lock: */
3653	if ((i & 1023) == 1023 && i + 1 < cPages)
3654	{
3655	PGM_UNLOCK(pVM);
3656	PGM_LOCK_VOID(pVM);
3657	}
3658	}
3659	}
3660	PGM_UNLOCK(pVM);
3661
3662	RT_BZERO(paLocks, sizeof(paLocks[0]) * cPages);
3663	}
3664	#endif /* IN_RING3 */
3665
3666
3667	/**
3668	* Release the internal mapping of a guest page.
3669	*
3670	* This is the counter part of pgmPhysGCPhys2CCPtrInternalEx and
3671	* pgmPhysGCPhys2CCPtrInternalReadOnly.
3672	*
3673	* @param pVM The cross context VM structure.
3674	* @param pLock The lock structure initialized by the mapping function.
3675	*
3676	* @remarks Caller must hold the PGM lock.
3677	*/
3678	void pgmPhysReleaseInternalPageMappingLock(PVMCC pVM, PPGMPAGEMAPLOCK pLock)
3679	{
3680	PGM_LOCK_ASSERT_OWNER(pVM);
3681	PGMPhysReleasePageMappingLock(pVM, pLock); /* lazy for now */
3682	}
3683
3684
3685	/**
3686	* Converts a GC physical address to a HC ring-3 pointer.
3687	*
3688	* @returns VINF_SUCCESS on success.
3689	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
3690	* page but has no physical backing.
3691	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
3692	* GC physical address.
3693	* @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
3694	* a dynamic ram chunk boundary
3695	*
3696	* @param pVM The cross context VM structure.
3697	* @param GCPhys The GC physical address to convert.
3698	* @param pR3Ptr Where to store the R3 pointer on success.
3699	*
3700	* @deprecated Avoid when possible!
3701	*/
3702	int pgmPhysGCPhys2R3Ptr(PVMCC pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
3703	{
3704	/** @todo this is kind of hacky and needs some more work. */
3705	#ifndef DEBUG_sandervl
3706	VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
3707	#endif
3708
3709	Log(("pgmPhysGCPhys2R3Ptr(,%RGp,): dont use this API!\n", GCPhys)); /** @todo eliminate this API! */
3710	PGM_LOCK_VOID(pVM);
3711
3712	PPGMRAMRANGE pRam;
3713	PPGMPAGE pPage;
3714	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
3715	if (RT_SUCCESS(rc))
3716	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr);
3717
3718	PGM_UNLOCK(pVM);
3719	Assert(rc <= VINF_SUCCESS);
3720	return rc;
3721	}
3722
3723
3724	/**
3725	* Special lockless guest physical to current context pointer convertor.
3726	*
3727	* This is mainly for the page table walking and such.
3728	*/
3729	int pgmPhysGCPhys2CCPtrLockless(PVMCPUCC pVCpu, RTGCPHYS GCPhys, void **ppv)
3730	{
3731	VMCPU_ASSERT_EMT(pVCpu);
3732
3733	/*
3734	* Get the RAM range and page structure.
3735	*/
3736	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3737	PGMRAMRANGE volatile *pRam;
3738	PGMPAGE volatile *pPage;
3739	int rc = pgmPhysGetPageAndRangeExLockless(pVM, pVCpu, GCPhys, &pPage, &pRam);
3740	if (RT_SUCCESS(rc))
3741	{
3742	/*
3743	* Now, make sure it's writable (typically it is).
3744	*/
3745	if (RT_LIKELY(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED))
3746	{ /* likely, typically */ }
3747	else
3748	{
3749	PGM_LOCK_VOID(pVM);
3750	rc = pgmPhysPageMakeWritable(pVM, (PPGMPAGE)pPage, GCPhys);
3751	if (RT_SUCCESS(rc))
3752	rc = pgmPhysGetPageAndRangeExLockless(pVM, pVCpu, GCPhys, &pPage, &pRam);
3753	PGM_UNLOCK(pVM);
3754	if (RT_FAILURE(rc))
3755	return rc;
3756	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
3757	}
3758	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0 \|\| PGM_IS_IN_NEM_MODE(pVM));
3759
3760	/*
3761	* Get the mapping address.
3762	*/
3763	uint8_t *pb;
3764	#ifdef IN_RING3
3765	if (PGM_IS_IN_NEM_MODE(pVM))
3766	pb = &pRam->pbR3[(RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << GUEST_PAGE_SHIFT];
3767	else
3768	#endif
3769	{
3770	#ifdef IN_RING3
3771	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
3772	PPGMPAGEMAPTLBE pTlbe;
3773	rc = pgmPhysPageQueryLocklessTlbeWithPage(pVCpu, (PPGMPAGE)pPage, GCPhys, &pTlbe);
3774	AssertLogRelRCReturn(rc, rc);
3775	pb = (uint8_t *)pTlbe->pv;
3776	RT_NOREF(pVM);
3777	# endif
3778	#else /** @todo a safe lockless page TLB in ring-0 needs the to ensure it gets the right invalidations. later. */
3779	PGM_LOCK(pVM);
3780	PPGMPAGEMAPTLBE pTlbe;
3781	rc = pgmPhysPageQueryTlbeWithPage(pVM, (PPGMPAGE)pPage, GCPhys, &pTlbe);
3782	AssertLogRelRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
3783	pb = (uint8_t *)pTlbe->pv;
3784	PGM_UNLOCK(pVM);
3785	RT_NOREF(pVCpu);
3786	#endif
3787	}
3788	ppv = (void )((uintptr_t)pb \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3789	return VINF_SUCCESS;
3790	}
3791	Assert(rc <= VINF_SUCCESS);
3792	return rc;
3793	}
3794
3795
3796	/**
3797	* Converts a guest pointer to a GC physical address.
3798	*
3799	* This uses the current CR3/CR0/CR4 of the guest.
3800	*
3801	* @returns VBox status code.
3802	* @param pVCpu The cross context virtual CPU structure.
3803	* @param GCPtr The guest pointer to convert.
3804	* @param pGCPhys Where to store the GC physical address.
3805	* @thread EMT(pVCpu)
3806	*/
3807	VMMDECL(int) PGMPhysGCPtr2GCPhys(PVMCPUCC pVCpu, RTGCPTR GCPtr, PRTGCPHYS pGCPhys)
3808	{
3809	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
3810	PGMPTWALK Walk;
3811	int rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtr, &Walk);
3812	if (pGCPhys && RT_SUCCESS(rc))
3813	*pGCPhys = Walk.GCPhys \| ((RTGCUINTPTR)GCPtr & GUEST_PAGE_OFFSET_MASK);
3814	return rc;
3815	}
3816
3817
3818	/**
3819	* Converts a guest pointer to a HC physical address.
3820	*
3821	* This uses the current CR3/CR0/CR4 of the guest.
3822	*
3823	* @returns VBox status code.
3824	* @param pVCpu The cross context virtual CPU structure.
3825	* @param GCPtr The guest pointer to convert.
3826	* @param pHCPhys Where to store the HC physical address.
3827	* @thread EMT(pVCpu)
3828	*/
3829	VMM_INT_DECL(int) PGMPhysGCPtr2HCPhys(PVMCPUCC pVCpu, RTGCPTR GCPtr, PRTHCPHYS pHCPhys)
3830	{
3831	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
3832	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3833	PGMPTWALK Walk;
3834	int rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtr, &Walk);
3835	if (RT_SUCCESS(rc))
3836	rc = PGMPhysGCPhys2HCPhys(pVM, Walk.GCPhys \| ((RTGCUINTPTR)GCPtr & GUEST_PAGE_OFFSET_MASK), pHCPhys);
3837	return rc;
3838	}
3839
3840
3841
3842	#undef LOG_GROUP
3843	#define LOG_GROUP LOG_GROUP_PGM_PHYS_ACCESS
3844
3845
3846	#if defined(IN_RING3) && defined(SOME_UNUSED_FUNCTION)
3847	/**
3848	* Cache PGMPhys memory access
3849	*
3850	* @param pVM The cross context VM structure.
3851	* @param pCache Cache structure pointer
3852	* @param GCPhys GC physical address
3853	* @param pbR3 HC pointer corresponding to physical page
3854	*
3855	* @thread EMT.
3856	*/
3857	static void pgmPhysCacheAdd(PVM pVM, PGMPHYSCACHE pCache, RTGCPHYS GCPhys, uint8_t pbR3)
3858	{
3859	uint32_t iCacheIndex;
3860
3861	Assert(VM_IS_EMT(pVM));
3862
3863	GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
3864	pbR3 = (uint8_t *)((uintptr_t)pbR3 & ~(uintptr_t)GUEST_PAGE_OFFSET_MASK);
3865
3866	iCacheIndex = ((GCPhys >> GUEST_PAGE_SHIFT) & PGM_MAX_PHYSCACHE_ENTRIES_MASK);
3867
3868	ASMBitSet(&pCache->aEntries, iCacheIndex);
3869
3870	pCache->Entry[iCacheIndex].GCPhys = GCPhys;
3871	pCache->Entry[iCacheIndex].pbR3 = pbR3;
3872	}
3873	#endif /* IN_RING3 */
3874
3875
3876	/**
3877	* Deals with reading from a page with one or more ALL access handlers.
3878	*
3879	* @returns Strict VBox status code in ring-0 and raw-mode, ignorable in ring-3.
3880	* See PGM_HANDLER_PHYS_IS_VALID_STATUS and
3881	* PGM_HANDLER_VIRT_IS_VALID_STATUS for details.
3882	*
3883	* @param pVM The cross context VM structure.
3884	* @param pPage The page descriptor.
3885	* @param GCPhys The physical address to start reading at.
3886	* @param pvBuf Where to put the bits we read.
3887	* @param cb How much to read - less or equal to a page.
3888	* @param enmOrigin The origin of this call.
3889	*/
3890	static VBOXSTRICTRC pgmPhysReadHandler(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void *pvBuf, size_t cb,
3891	PGMACCESSORIGIN enmOrigin)
3892	{
3893	/*
3894	* The most frequent access here is MMIO and shadowed ROM.
3895	* The current code ASSUMES all these access handlers covers full pages!
3896	*/
3897
3898	/*
3899	* Whatever we do we need the source page, map it first.
3900	*/
3901	PGMPAGEMAPLOCK PgMpLck;
3902	const void *pvSrc = NULL;
3903	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvSrc, &PgMpLck);
3904	/** @todo Check how this can work for MMIO pages? */
3905	if (RT_FAILURE(rc))
3906	{
3907	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
3908	GCPhys, pPage, rc));
3909	memset(pvBuf, 0xff, cb);
3910	return VINF_SUCCESS;
3911	}
3912
3913	VBOXSTRICTRC rcStrict = VINF_PGM_HANDLER_DO_DEFAULT;
3914
3915	/*
3916	* Deal with any physical handlers.
3917	*/
3918	PVMCPUCC pVCpu = VMMGetCpu(pVM);
3919	if ( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL
3920	\|\| PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
3921	{
3922	PPGMPHYSHANDLER pCur;
3923	rc = pgmHandlerPhysicalLookup(pVM, GCPhys, &pCur);
3924	if (RT_SUCCESS(rc))
3925	{
3926	Assert(pCur && GCPhys >= pCur->Key && GCPhys <= pCur->KeyLast);
3927	Assert((pCur->Key & GUEST_PAGE_OFFSET_MASK) == 0);
3928	Assert((pCur->KeyLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK);
3929	#ifndef IN_RING3
3930	if (enmOrigin != PGMACCESSORIGIN_IEM)
3931	{
3932	/* Cannot reliably handle informational status codes in this context */
3933	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
3934	return VERR_PGM_PHYS_WR_HIT_HANDLER;
3935	}
3936	#endif
3937	PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
3938	PFNPGMPHYSHANDLER const pfnHandler = pCurType->pfnHandler; Assert(pfnHandler);
3939	uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
3940	: (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
3941
3942	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cb, pPage, R3STRING(pCur->pszDesc) ));
3943	STAM_PROFILE_START(&pCur->Stat, h);
3944	PGM_LOCK_ASSERT_OWNER(pVM);
3945
3946	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
3947	PGM_UNLOCK(pVM);
3948	/* If the access origins with a device, make sure the buffer is initialized
3949	as a guard against leaking heap, stack and other info via badly written
3950	MMIO handling. @bugref{10651} */
3951	if (enmOrigin == PGMACCESSORIGIN_DEVICE)
3952	memset(pvBuf, 0xff, cb);
3953	rcStrict = pfnHandler(pVM, pVCpu, GCPhys, (void *)pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, enmOrigin, uUser);
3954	PGM_LOCK_VOID(pVM);
3955
3956	STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
3957	pCur = NULL; /* might not be valid anymore. */
3958	AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict, false),
3959	("rcStrict=%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys));
3960	if ( rcStrict != VINF_PGM_HANDLER_DO_DEFAULT
3961	&& !PGM_PHYS_RW_IS_SUCCESS(rcStrict))
3962	{
3963	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
3964	return rcStrict;
3965	}
3966	}
3967	else if (rc == VERR_NOT_FOUND)
3968	AssertLogRelMsgFailed(("rc=%Rrc GCPhys=%RGp cb=%#x\n", rc, GCPhys, cb));
3969	else
3970	AssertLogRelMsgFailedReturn(("rc=%Rrc GCPhys=%RGp cb=%#x\n", rc, GCPhys, cb), rc);
3971	}
3972
3973	/*
3974	* Take the default action.
3975	*/
3976	if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
3977	{
3978	memcpy(pvBuf, pvSrc, cb);
3979	rcStrict = VINF_SUCCESS;
3980	}
3981	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
3982	return rcStrict;
3983	}
3984
3985
3986	/**
3987	* Read physical memory.
3988	*
3989	* This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
3990	* want to ignore those.
3991	*
3992	* @returns Strict VBox status code in raw-mode and ring-0, normal VBox status
3993	* code in ring-3. Use PGM_PHYS_RW_IS_SUCCESS to check.
3994	* @retval VINF_SUCCESS in all context - read completed.
3995	*
3996	* @retval VINF_EM_OFF in RC and R0 - read completed.
3997	* @retval VINF_EM_SUSPEND in RC and R0 - read completed.
3998	* @retval VINF_EM_RESET in RC and R0 - read completed.
3999	* @retval VINF_EM_HALT in RC and R0 - read completed.
4000	* @retval VINF_SELM_SYNC_GDT in RC only - read completed.
4001	*
4002	* @retval VINF_EM_DBG_STOP in RC and R0 - read completed.
4003	* @retval VINF_EM_DBG_BREAKPOINT in RC and R0 - read completed.
4004	* @retval VINF_EM_RAW_EMULATE_INSTR in RC and R0 only.
4005	*
4006	* @retval VINF_IOM_R3_MMIO_READ in RC and R0.
4007	* @retval VINF_IOM_R3_MMIO_READ_WRITE in RC and R0.
4008	*
4009	* @retval VINF_PATM_CHECK_PATCH_PAGE in RC only.
4010	*
4011	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in RC and R0 for access origins that
4012	* haven't been cleared for strict status codes yet.
4013	*
4014	* @param pVM The cross context VM structure.
4015	* @param GCPhys Physical address start reading from.
4016	* @param pvBuf Where to put the read bits.
4017	* @param cbRead How many bytes to read.
4018	* @param enmOrigin The origin of this call.
4019	*/
4020	VMMDECL(VBOXSTRICTRC) PGMPhysRead(PVMCC pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead, PGMACCESSORIGIN enmOrigin)
4021	{
4022	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
4023	LogFlow(("PGMPhysRead: %RGp %d\n", GCPhys, cbRead));
4024
4025	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysRead));
4026	STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysReadBytes), cbRead);
4027
4028	PGM_LOCK_VOID(pVM);
4029
4030	/*
4031	* Copy loop on ram ranges.
4032	*/
4033	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
4034	for (;;)
4035	{
4036	PPGMRAMRANGE const pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
4037
4038	/* Inside range or not? */
4039	if (pRam && GCPhys >= pRam->GCPhys)
4040	{
4041	/*
4042	* Must work our way thru this page by page.
4043	*/
4044	RTGCPHYS off = GCPhys - pRam->GCPhys;
4045	while (off < pRam->cb)
4046	{
4047	unsigned iPage = off >> GUEST_PAGE_SHIFT;
4048	PPGMPAGE pPage = &pRam->aPages[iPage];
4049	size_t cb = GUEST_PAGE_SIZE - (off & GUEST_PAGE_OFFSET_MASK);
4050	if (cb > cbRead)
4051	cb = cbRead;
4052
4053	/*
4054	* Normal page? Get the pointer to it.
4055	*/
4056	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
4057	&& !PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
4058	{
4059	/*
4060	* Get the pointer to the page.
4061	*/
4062	PGMPAGEMAPLOCK PgMpLck;
4063	const void *pvSrc;
4064	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
4065	if (RT_SUCCESS(rc))
4066	{
4067	memcpy(pvBuf, pvSrc, cb);
4068	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
4069	}
4070	else
4071	{
4072	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
4073	pRam->GCPhys + off, pPage, rc));
4074	memset(pvBuf, 0xff, cb);
4075	}
4076	}
4077	/*
4078	* Have ALL/MMIO access handlers.
4079	*/
4080	else
4081	{
4082	VBOXSTRICTRC rcStrict2 = pgmPhysReadHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb, enmOrigin);
4083	if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
4084	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
4085	else
4086	{
4087	/* Set the remaining buffer to a known value. */
4088	memset(pvBuf, 0xff, cbRead);
4089	PGM_UNLOCK(pVM);
4090	return rcStrict2;
4091	}
4092	}
4093
4094	/* next page */
4095	if (cb >= cbRead)
4096	{
4097	PGM_UNLOCK(pVM);
4098	return rcStrict;
4099	}
4100	cbRead -= cb;
4101	off += cb;
4102	pvBuf = (char *)pvBuf + cb;
4103	} /* walk pages in ram range. */
4104
4105	GCPhys = pRam->GCPhysLast + 1;
4106	}
4107	else
4108	{
4109	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
4110
4111	/*
4112	* Unassigned address space.
4113	*/
4114	size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
4115	if (cb >= cbRead)
4116	{
4117	memset(pvBuf, 0xff, cbRead);
4118	break;
4119	}
4120	memset(pvBuf, 0xff, cb);
4121
4122	cbRead -= cb;
4123	pvBuf = (char *)pvBuf + cb;
4124	GCPhys += cb;
4125	}
4126
4127	} /* Ram range walk */
4128
4129	PGM_UNLOCK(pVM);
4130	return rcStrict;
4131	}
4132
4133
4134	/**
4135	* Deals with writing to a page with one or more WRITE or ALL access handlers.
4136	*
4137	* @returns Strict VBox status code in ring-0 and raw-mode, ignorable in ring-3.
4138	* See PGM_HANDLER_PHYS_IS_VALID_STATUS and
4139	* PGM_HANDLER_VIRT_IS_VALID_STATUS for details.
4140	*
4141	* @param pVM The cross context VM structure.
4142	* @param pPage The page descriptor.
4143	* @param GCPhys The physical address to start writing at.
4144	* @param pvBuf What to write.
4145	* @param cbWrite How much to write - less or equal to a page.
4146	* @param enmOrigin The origin of this call.
4147	*/
4148	static VBOXSTRICTRC pgmPhysWriteHandler(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const *pvBuf, size_t cbWrite,
4149	PGMACCESSORIGIN enmOrigin)
4150	{
4151	PGMPAGEMAPLOCK PgMpLck;
4152	void *pvDst = NULL;
4153	VBOXSTRICTRC rcStrict;
4154
4155	/*
4156	* Give priority to physical handlers (like #PF does).
4157	*
4158	* Hope for a lonely physical handler first that covers the whole write
4159	* area. This should be a pretty frequent case with MMIO and the heavy
4160	* usage of full page handlers in the page pool.
4161	*/
4162	PVMCPUCC pVCpu = VMMGetCpu(pVM);
4163	PPGMPHYSHANDLER pCur;
4164	rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhys, &pCur);
4165	if (RT_SUCCESS(rcStrict))
4166	{
4167	Assert(GCPhys >= pCur->Key && GCPhys <= pCur->KeyLast);
4168	#ifndef IN_RING3
4169	if (enmOrigin != PGMACCESSORIGIN_IEM)
4170	/* Cannot reliably handle informational status codes in this context */
4171	return VERR_PGM_PHYS_WR_HIT_HANDLER;
4172	#endif
4173	size_t cbRange = pCur->KeyLast - GCPhys + 1;
4174	if (cbRange > cbWrite)
4175	cbRange = cbWrite;
4176
4177	Assert(PGMPHYSHANDLER_GET_TYPE(pVM, pCur)->pfnHandler);
4178	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n",
4179	GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
4180	if (!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
4181	rcStrict = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
4182	else
4183	rcStrict = VINF_SUCCESS;
4184	if (RT_SUCCESS(rcStrict))
4185	{
4186	PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
4187	PFNPGMPHYSHANDLER const pfnHandler = pCurType->pfnHandler;
4188	uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
4189	: (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
4190	STAM_PROFILE_START(&pCur->Stat, h);
4191
4192	/* Most handlers will want to release the PGM lock for deadlock prevention
4193	(esp. MMIO), though some PGM internal ones like the page pool and MMIO2
4194	dirty page trackers will want to keep it for performance reasons. */
4195	PGM_LOCK_ASSERT_OWNER(pVM);
4196	if (pCurType->fKeepPgmLock)
4197	rcStrict = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser);
4198	else
4199	{
4200	PGM_UNLOCK(pVM);
4201	rcStrict = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser);
4202	PGM_LOCK_VOID(pVM);
4203	}
4204
4205	STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
4206	pCur = NULL; /* might not be valid anymore. */
4207	if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
4208	{
4209	if (pvDst)
4210	memcpy(pvDst, pvBuf, cbRange);
4211	rcStrict = VINF_SUCCESS;
4212	}
4213	else
4214	AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict, true),
4215	("rcStrict=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n",
4216	VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pCur ? R3STRING(pCur->pszDesc) : ""));
4217	}
4218	else
4219	AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
4220	GCPhys, pPage, VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
4221	if (RT_LIKELY(cbRange == cbWrite) \|\| !PGM_PHYS_RW_IS_SUCCESS(rcStrict))
4222	{
4223	if (pvDst)
4224	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
4225	return rcStrict;
4226	}
4227
4228	/* more fun to be had below */
4229	cbWrite -= cbRange;
4230	GCPhys += cbRange;
4231	pvBuf = (uint8_t *)pvBuf + cbRange;
4232	pvDst = (uint8_t *)pvDst + cbRange;
4233	}
4234	else if (rcStrict == VERR_NOT_FOUND) /* The handler is somewhere else in the page, deal with it below. */
4235	rcStrict = VINF_SUCCESS;
4236	else
4237	AssertMsgFailedReturn(("rcStrict=%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys), rcStrict);
4238	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)); /* MMIO handlers are all GUEST_PAGE_SIZEed! */
4239
4240	/*
4241	* Deal with all the odd ends (used to be deal with virt+phys).
4242	*/
4243	Assert(rcStrict != VINF_PGM_HANDLER_DO_DEFAULT);
4244
4245	/* We need a writable destination page. */
4246	if (!pvDst)
4247	{
4248	int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
4249	AssertLogRelMsgReturn(RT_SUCCESS(rc2),
4250	("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n", GCPhys, pPage, rc2),
4251	rc2);
4252	}
4253
4254	/** @todo clean up this code some more now there are no virtual handlers any
4255	* more. */
4256	/* The loop state (big + ugly). */
4257	PPGMPHYSHANDLER pPhys = NULL;
4258	uint32_t offPhys = GUEST_PAGE_SIZE;
4259	uint32_t offPhysLast = GUEST_PAGE_SIZE;
4260	bool fMorePhys = PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage);
4261
4262	/* The loop. */
4263	for (;;)
4264	{
4265	if (fMorePhys && !pPhys)
4266	{
4267	rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhys, &pPhys);
4268	if (RT_SUCCESS_NP(rcStrict))
4269	{
4270	offPhys = 0;
4271	offPhysLast = pPhys->KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
4272	}
4273	else
4274	{
4275	AssertMsgReturn(rcStrict == VERR_NOT_FOUND, ("%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys), rcStrict);
4276
4277	rcStrict = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookupMatchingOrAbove(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator,
4278	GCPhys, &pPhys);
4279	AssertMsgReturn(RT_SUCCESS(rcStrict) \|\| rcStrict == VERR_NOT_FOUND,
4280	("%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys), rcStrict);
4281
4282	if ( RT_SUCCESS(rcStrict)
4283	&& pPhys->Key <= GCPhys + (cbWrite - 1))
4284	{
4285	offPhys = pPhys->Key - GCPhys;
4286	offPhysLast = pPhys->KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
4287	Assert(pPhys->KeyLast - pPhys->Key < _4G);
4288	}
4289	else
4290	{
4291	pPhys = NULL;
4292	fMorePhys = false;
4293	offPhys = offPhysLast = GUEST_PAGE_SIZE;
4294	}
4295	}
4296	}
4297
4298	/*
4299	* Handle access to space without handlers (that's easy).
4300	*/
4301	VBOXSTRICTRC rcStrict2 = VINF_PGM_HANDLER_DO_DEFAULT;
4302	uint32_t cbRange = (uint32_t)cbWrite;
4303	Assert(cbRange == cbWrite);
4304
4305	/*
4306	* Physical handler.
4307	*/
4308	if (!offPhys)
4309	{
4310	#ifndef IN_RING3
4311	if (enmOrigin != PGMACCESSORIGIN_IEM)
4312	/* Cannot reliably handle informational status codes in this context */
4313	return VERR_PGM_PHYS_WR_HIT_HANDLER;
4314	#endif
4315	if (cbRange > offPhysLast + 1)
4316	cbRange = offPhysLast + 1;
4317
4318	PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pPhys);
4319	PFNPGMPHYSHANDLER const pfnHandler = pCurType->pfnHandler;
4320	uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pPhys->uUser
4321	: (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pPhys->uUser);
4322
4323	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc) ));
4324	STAM_PROFILE_START(&pPhys->Stat, h);
4325
4326	/* Most handlers will want to release the PGM lock for deadlock prevention
4327	(esp. MMIO), though some PGM internal ones like the page pool and MMIO2
4328	dirty page trackers will want to keep it for performance reasons. */
4329	PGM_LOCK_ASSERT_OWNER(pVM);
4330	if (pCurType->fKeepPgmLock)
4331	rcStrict2 = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser);
4332	else
4333	{
4334	PGM_UNLOCK(pVM);
4335	rcStrict2 = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser);
4336	PGM_LOCK_VOID(pVM);
4337	}
4338
4339	STAM_PROFILE_STOP(&pPhys->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
4340	pPhys = NULL; /* might not be valid anymore. */
4341	AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict2, true),
4342	("rcStrict2=%Rrc (rcStrict=%Rrc) GCPhys=%RGp pPage=%R[pgmpage] %s\n", VBOXSTRICTRC_VAL(rcStrict2),
4343	VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pPhys ? R3STRING(pPhys->pszDesc) : ""));
4344	}
4345
4346	/*
4347	* Execute the default action and merge the status codes.
4348	*/
4349	if (rcStrict2 == VINF_PGM_HANDLER_DO_DEFAULT)
4350	{
4351	memcpy(pvDst, pvBuf, cbRange);
4352	rcStrict2 = VINF_SUCCESS;
4353	}
4354	else if (!PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
4355	{
4356	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
4357	return rcStrict2;
4358	}
4359	else
4360	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
4361
4362	/*
4363	* Advance if we've got more stuff to do.
4364	*/
4365	if (cbRange >= cbWrite)
4366	{
4367	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
4368	return rcStrict;
4369	}
4370
4371
4372	cbWrite -= cbRange;
4373	GCPhys += cbRange;
4374	pvBuf = (uint8_t *)pvBuf + cbRange;
4375	pvDst = (uint8_t *)pvDst + cbRange;
4376
4377	offPhys -= cbRange;
4378	offPhysLast -= cbRange;
4379	}
4380	}
4381
4382
4383	/**
4384	* Write to physical memory.
4385	*
4386	* This API respects access handlers and MMIO. Use PGMPhysSimpleWriteGCPhys() if you
4387	* want to ignore those.
4388	*
4389	* @returns Strict VBox status code in raw-mode and ring-0, normal VBox status
4390	* code in ring-3. Use PGM_PHYS_RW_IS_SUCCESS to check.
4391	* @retval VINF_SUCCESS in all context - write completed.
4392	*
4393	* @retval VINF_EM_OFF in RC and R0 - write completed.
4394	* @retval VINF_EM_SUSPEND in RC and R0 - write completed.
4395	* @retval VINF_EM_RESET in RC and R0 - write completed.
4396	* @retval VINF_EM_HALT in RC and R0 - write completed.
4397	* @retval VINF_SELM_SYNC_GDT in RC only - write completed.
4398	*
4399	* @retval VINF_EM_DBG_STOP in RC and R0 - write completed.
4400	* @retval VINF_EM_DBG_BREAKPOINT in RC and R0 - write completed.
4401	* @retval VINF_EM_RAW_EMULATE_INSTR in RC and R0 only.
4402	*
4403	* @retval VINF_IOM_R3_MMIO_WRITE in RC and R0.
4404	* @retval VINF_IOM_R3_MMIO_READ_WRITE in RC and R0.
4405	* @retval VINF_IOM_R3_MMIO_COMMIT_WRITE in RC and R0.
4406	*
4407	* @retval VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT in RC only - write completed.
4408	* @retval VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT in RC only.
4409	* @retval VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT in RC only.
4410	* @retval VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT in RC only.
4411	* @retval VINF_CSAM_PENDING_ACTION in RC only.
4412	* @retval VINF_PATM_CHECK_PATCH_PAGE in RC only.
4413	*
4414	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in RC and R0 for access origins that
4415	* haven't been cleared for strict status codes yet.
4416	*
4417	*
4418	* @param pVM The cross context VM structure.
4419	* @param GCPhys Physical address to write to.
4420	* @param pvBuf What to write.
4421	* @param cbWrite How many bytes to write.
4422	* @param enmOrigin Who is calling.
4423	*/
4424	VMMDECL(VBOXSTRICTRC) PGMPhysWrite(PVMCC pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, PGMACCESSORIGIN enmOrigin)
4425	{
4426	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites, ("Calling PGMPhysWrite after pgmR3Save()! enmOrigin=%d\n", enmOrigin));
4427	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
4428	LogFlow(("PGMPhysWrite: %RGp %d\n", GCPhys, cbWrite));
4429
4430	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysWrite));
4431	STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysWriteBytes), cbWrite);
4432
4433	PGM_LOCK_VOID(pVM);
4434
4435	/*
4436	* Copy loop on ram ranges.
4437	*/
4438	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
4439	for (;;)
4440	{
4441	PPGMRAMRANGE const pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
4442
4443	/* Inside range or not? */
4444	if (pRam && GCPhys >= pRam->GCPhys)
4445	{
4446	/*
4447	* Must work our way thru this page by page.
4448	*/
4449	RTGCPTR off = GCPhys - pRam->GCPhys;
4450	while (off < pRam->cb)
4451	{
4452	RTGCPTR iPage = off >> GUEST_PAGE_SHIFT;
4453	PPGMPAGE pPage = &pRam->aPages[iPage];
4454	size_t cb = GUEST_PAGE_SIZE - (off & GUEST_PAGE_OFFSET_MASK);
4455	if (cb > cbWrite)
4456	cb = cbWrite;
4457
4458	/*
4459	* Normal page? Get the pointer to it.
4460	*/
4461	if ( !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
4462	&& !PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
4463	{
4464	PGMPAGEMAPLOCK PgMpLck;
4465	void *pvDst;
4466	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
4467	if (RT_SUCCESS(rc))
4468	{
4469	Assert(!PGM_PAGE_IS_BALLOONED(pPage));
4470	memcpy(pvDst, pvBuf, cb);
4471	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
4472	}
4473	/* Ignore writes to ballooned pages. */
4474	else if (!PGM_PAGE_IS_BALLOONED(pPage))
4475	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
4476	pRam->GCPhys + off, pPage, rc));
4477	}
4478	/*
4479	* Active WRITE or ALL access handlers.
4480	*/
4481	else
4482	{
4483	VBOXSTRICTRC rcStrict2 = pgmPhysWriteHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb, enmOrigin);
4484	if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
4485	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
4486	else
4487	{
4488	PGM_UNLOCK(pVM);
4489	return rcStrict2;
4490	}
4491	}
4492
4493	/* next page */
4494	if (cb >= cbWrite)
4495	{
4496	PGM_UNLOCK(pVM);
4497	return rcStrict;
4498	}
4499
4500	cbWrite -= cb;
4501	off += cb;
4502	pvBuf = (const char *)pvBuf + cb;
4503	} /* walk pages in ram range */
4504
4505	GCPhys = pRam->GCPhysLast + 1;
4506	}
4507	else
4508	{
4509	/*
4510	* Unassigned address space, skip it.
4511	*/
4512	if (!pRam)
4513	break;
4514	size_t cb = pRam->GCPhys - GCPhys;
4515	if (cb >= cbWrite)
4516	break;
4517	cbWrite -= cb;
4518	pvBuf = (const char *)pvBuf + cb;
4519	GCPhys += cb;
4520	}
4521
4522	} /* Ram range walk */
4523
4524	PGM_UNLOCK(pVM);
4525	return rcStrict;
4526	}
4527
4528
4529	/**
4530	* Read from guest physical memory by GC physical address, bypassing
4531	* MMIO and access handlers.
4532	*
4533	* @returns VBox status code.
4534	* @param pVM The cross context VM structure.
4535	* @param pvDst The destination address.
4536	* @param GCPhysSrc The source address (GC physical address).
4537	* @param cb The number of bytes to read.
4538	*/
4539	VMMDECL(int) PGMPhysSimpleReadGCPhys(PVMCC pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb)
4540	{
4541	/*
4542	* Treat the first page as a special case.
4543	*/
4544	if (!cb)
4545	return VINF_SUCCESS;
4546
4547	/* map the 1st page */
4548	void const *pvSrc;
4549	PGMPAGEMAPLOCK Lock;
4550	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
4551	if (RT_FAILURE(rc))
4552	return rc;
4553
4554	/* optimize for the case where access is completely within the first page. */
4555	size_t cbPage = GUEST_PAGE_SIZE - (GCPhysSrc & GUEST_PAGE_OFFSET_MASK);
4556	if (RT_LIKELY(cb <= cbPage))
4557	{
4558	memcpy(pvDst, pvSrc, cb);
4559	PGMPhysReleasePageMappingLock(pVM, &Lock);
4560	return VINF_SUCCESS;
4561	}
4562
4563	/* copy to the end of the page. */
4564	memcpy(pvDst, pvSrc, cbPage);
4565	PGMPhysReleasePageMappingLock(pVM, &Lock);
4566	GCPhysSrc += cbPage;
4567	pvDst = (uint8_t *)pvDst + cbPage;
4568	cb -= cbPage;
4569
4570	/*
4571	* Page by page.
4572	*/
4573	for (;;)
4574	{
4575	/* map the page */
4576	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
4577	if (RT_FAILURE(rc))
4578	return rc;
4579
4580	/* last page? */
4581	if (cb <= GUEST_PAGE_SIZE)
4582	{
4583	memcpy(pvDst, pvSrc, cb);
4584	PGMPhysReleasePageMappingLock(pVM, &Lock);
4585	return VINF_SUCCESS;
4586	}
4587
4588	/* copy the entire page and advance */
4589	memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
4590	PGMPhysReleasePageMappingLock(pVM, &Lock);
4591	GCPhysSrc += GUEST_PAGE_SIZE;
4592	pvDst = (uint8_t *)pvDst + GUEST_PAGE_SIZE;
4593	cb -= GUEST_PAGE_SIZE;
4594	}
4595	/* won't ever get here. */
4596	}
4597
4598
4599	/**
4600	* Write to guest physical memory referenced by GC pointer.
4601	* Write memory to GC physical address in guest physical memory.
4602	*
4603	* This will bypass MMIO and access handlers.
4604	*
4605	* @returns VBox status code.
4606	* @param pVM The cross context VM structure.
4607	* @param GCPhysDst The GC physical address of the destination.
4608	* @param pvSrc The source buffer.
4609	* @param cb The number of bytes to write.
4610	*/
4611	VMMDECL(int) PGMPhysSimpleWriteGCPhys(PVMCC pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb)
4612	{
4613	LogFlow(("PGMPhysSimpleWriteGCPhys: %RGp %zu\n", GCPhysDst, cb));
4614
4615	/*
4616	* Treat the first page as a special case.
4617	*/
4618	if (!cb)
4619	return VINF_SUCCESS;
4620
4621	/* map the 1st page */
4622	void *pvDst;
4623	PGMPAGEMAPLOCK Lock;
4624	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
4625	if (RT_FAILURE(rc))
4626	return rc;
4627
4628	/* optimize for the case where access is completely within the first page. */
4629	size_t cbPage = GUEST_PAGE_SIZE - (GCPhysDst & GUEST_PAGE_OFFSET_MASK);
4630	if (RT_LIKELY(cb <= cbPage))
4631	{
4632	memcpy(pvDst, pvSrc, cb);
4633	PGMPhysReleasePageMappingLock(pVM, &Lock);
4634	return VINF_SUCCESS;
4635	}
4636
4637	/* copy to the end of the page. */
4638	memcpy(pvDst, pvSrc, cbPage);
4639	PGMPhysReleasePageMappingLock(pVM, &Lock);
4640	GCPhysDst += cbPage;
4641	pvSrc = (const uint8_t *)pvSrc + cbPage;
4642	cb -= cbPage;
4643
4644	/*
4645	* Page by page.
4646	*/
4647	for (;;)
4648	{
4649	/* map the page */
4650	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
4651	if (RT_FAILURE(rc))
4652	return rc;
4653
4654	/* last page? */
4655	if (cb <= GUEST_PAGE_SIZE)
4656	{
4657	memcpy(pvDst, pvSrc, cb);
4658	PGMPhysReleasePageMappingLock(pVM, &Lock);
4659	return VINF_SUCCESS;
4660	}
4661
4662	/* copy the entire page and advance */
4663	memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
4664	PGMPhysReleasePageMappingLock(pVM, &Lock);
4665	GCPhysDst += GUEST_PAGE_SIZE;
4666	pvSrc = (const uint8_t *)pvSrc + GUEST_PAGE_SIZE;
4667	cb -= GUEST_PAGE_SIZE;
4668	}
4669	/* won't ever get here. */
4670	}
4671
4672
4673	/**
4674	* Read from guest physical memory referenced by GC pointer.
4675	*
4676	* This function uses the current CR3/CR0/CR4 of the guest and will
4677	* bypass access handlers and not set any accessed bits.
4678	*
4679	* @returns VBox status code.
4680	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4681	* @param pvDst The destination address.
4682	* @param GCPtrSrc The source address (GC pointer).
4683	* @param cb The number of bytes to read.
4684	*/
4685	VMMDECL(int) PGMPhysSimpleReadGCPtr(PVMCPUCC pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
4686	{
4687	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4688	/** @todo fix the macro / state handling: VMCPU_ASSERT_EMT_OR_GURU(pVCpu); */
4689
4690	/*
4691	* Treat the first page as a special case.
4692	*/
4693	if (!cb)
4694	return VINF_SUCCESS;
4695
4696	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleRead));
4697	STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleReadBytes), cb);
4698
4699	/* Take the PGM lock here, because many called functions take the lock for a very short period. That's counter-productive
4700	* when many VCPUs are fighting for the lock.
4701	*/
4702	PGM_LOCK_VOID(pVM);
4703
4704	/* map the 1st page */
4705	void const *pvSrc;
4706	PGMPAGEMAPLOCK Lock;
4707	int rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
4708	if (RT_FAILURE(rc))
4709	{
4710	PGM_UNLOCK(pVM);
4711	return rc;
4712	}
4713
4714	/* optimize for the case where access is completely within the first page. */
4715	size_t cbPage = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
4716	if (RT_LIKELY(cb <= cbPage))
4717	{
4718	memcpy(pvDst, pvSrc, cb);
4719	PGMPhysReleasePageMappingLock(pVM, &Lock);
4720	PGM_UNLOCK(pVM);
4721	return VINF_SUCCESS;
4722	}
4723
4724	/* copy to the end of the page. */
4725	memcpy(pvDst, pvSrc, cbPage);
4726	PGMPhysReleasePageMappingLock(pVM, &Lock);
4727	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + cbPage);
4728	pvDst = (uint8_t *)pvDst + cbPage;
4729	cb -= cbPage;
4730
4731	/*
4732	* Page by page.
4733	*/
4734	for (;;)
4735	{
4736	/* map the page */
4737	rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
4738	if (RT_FAILURE(rc))
4739	{
4740	PGM_UNLOCK(pVM);
4741	return rc;
4742	}
4743
4744	/* last page? */
4745	if (cb <= GUEST_PAGE_SIZE)
4746	{
4747	memcpy(pvDst, pvSrc, cb);
4748	PGMPhysReleasePageMappingLock(pVM, &Lock);
4749	PGM_UNLOCK(pVM);
4750	return VINF_SUCCESS;
4751	}
4752
4753	/* copy the entire page and advance */
4754	memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
4755	PGMPhysReleasePageMappingLock(pVM, &Lock);
4756	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + GUEST_PAGE_SIZE);
4757	pvDst = (uint8_t *)pvDst + GUEST_PAGE_SIZE;
4758	cb -= GUEST_PAGE_SIZE;
4759	}
4760	/* won't ever get here. */
4761	}
4762
4763
4764	/**
4765	* Write to guest physical memory referenced by GC pointer.
4766	*
4767	* This function uses the current CR3/CR0/CR4 of the guest and will
4768	* bypass access handlers and not set dirty or accessed bits.
4769	*
4770	* @returns VBox status code.
4771	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4772	* @param GCPtrDst The destination address (GC pointer).
4773	* @param pvSrc The source address.
4774	* @param cb The number of bytes to write.
4775	*/
4776	VMMDECL(int) PGMPhysSimpleWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
4777	{
4778	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4779	VMCPU_ASSERT_EMT(pVCpu);
4780
4781	/*
4782	* Treat the first page as a special case.
4783	*/
4784	if (!cb)
4785	return VINF_SUCCESS;
4786
4787	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleWrite));
4788	STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleWriteBytes), cb);
4789
4790	/* map the 1st page */
4791	void *pvDst;
4792	PGMPAGEMAPLOCK Lock;
4793	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
4794	if (RT_FAILURE(rc))
4795	return rc;
4796
4797	/* optimize for the case where access is completely within the first page. */
4798	size_t cbPage = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
4799	if (RT_LIKELY(cb <= cbPage))
4800	{
4801	memcpy(pvDst, pvSrc, cb);
4802	PGMPhysReleasePageMappingLock(pVM, &Lock);
4803	return VINF_SUCCESS;
4804	}
4805
4806	/* copy to the end of the page. */
4807	memcpy(pvDst, pvSrc, cbPage);
4808	PGMPhysReleasePageMappingLock(pVM, &Lock);
4809	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
4810	pvSrc = (const uint8_t *)pvSrc + cbPage;
4811	cb -= cbPage;
4812
4813	/*
4814	* Page by page.
4815	*/
4816	for (;;)
4817	{
4818	/* map the page */
4819	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
4820	if (RT_FAILURE(rc))
4821	return rc;
4822
4823	/* last page? */
4824	if (cb <= GUEST_PAGE_SIZE)
4825	{
4826	memcpy(pvDst, pvSrc, cb);
4827	PGMPhysReleasePageMappingLock(pVM, &Lock);
4828	return VINF_SUCCESS;
4829	}
4830
4831	/* copy the entire page and advance */
4832	memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
4833	PGMPhysReleasePageMappingLock(pVM, &Lock);
4834	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + GUEST_PAGE_SIZE);
4835	pvSrc = (const uint8_t *)pvSrc + GUEST_PAGE_SIZE;
4836	cb -= GUEST_PAGE_SIZE;
4837	}
4838	/* won't ever get here. */
4839	}
4840
4841
4842	/**
4843	* Write to guest physical memory referenced by GC pointer and update the PTE.
4844	*
4845	* This function uses the current CR3/CR0/CR4 of the guest and will
4846	* bypass access handlers but will set any dirty and accessed bits in the PTE.
4847	*
4848	* If you don't want to set the dirty bit, use PGMPhysSimpleWriteGCPtr().
4849	*
4850	* @returns VBox status code.
4851	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4852	* @param GCPtrDst The destination address (GC pointer).
4853	* @param pvSrc The source address.
4854	* @param cb The number of bytes to write.
4855	*/
4856	VMMDECL(int) PGMPhysSimpleDirtyWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
4857	{
4858	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4859	VMCPU_ASSERT_EMT(pVCpu);
4860
4861	/*
4862	* Treat the first page as a special case.
4863	* Btw. this is the same code as in PGMPhyssimpleWriteGCPtr excep for the PGMGstModifyPage.
4864	*/
4865	if (!cb)
4866	return VINF_SUCCESS;
4867
4868	/* map the 1st page */
4869	void *pvDst;
4870	PGMPAGEMAPLOCK Lock;
4871	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
4872	if (RT_FAILURE(rc))
4873	return rc;
4874
4875	/* optimize for the case where access is completely within the first page. */
4876	size_t cbPage = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
4877	if (RT_LIKELY(cb <= cbPage))
4878	{
4879	memcpy(pvDst, pvSrc, cb);
4880	PGMPhysReleasePageMappingLock(pVM, &Lock);
4881	#ifdef VBOX_VMM_TARGET_X86
4882	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
4883	#elif !defined(VBOX_VMM_TARGET_ARMV8)
4884	# error "misconfig"
4885	#endif
4886	return VINF_SUCCESS;
4887	}
4888
4889	/* copy to the end of the page. */
4890	memcpy(pvDst, pvSrc, cbPage);
4891	PGMPhysReleasePageMappingLock(pVM, &Lock);
4892	#ifdef VBOX_VMM_TARGET_X86
4893	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
4894	#elif !defined(VBOX_VMM_TARGET_ARMV8)
4895	# error "misconfig"
4896	#endif
4897	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
4898	pvSrc = (const uint8_t *)pvSrc + cbPage;
4899	cb -= cbPage;
4900
4901	/*
4902	* Page by page.
4903	*/
4904	for (;;)
4905	{
4906	/* map the page */
4907	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
4908	if (RT_FAILURE(rc))
4909	return rc;
4910
4911	/* last page? */
4912	if (cb <= GUEST_PAGE_SIZE)
4913	{
4914	memcpy(pvDst, pvSrc, cb);
4915	PGMPhysReleasePageMappingLock(pVM, &Lock);
4916	#ifdef VBOX_VMM_TARGET_X86
4917	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
4918	#elif !defined(VBOX_VMM_TARGET_ARMV8)
4919	# error "misconfig"
4920	#endif
4921	return VINF_SUCCESS;
4922	}
4923
4924	/* copy the entire page and advance */
4925	memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
4926	PGMPhysReleasePageMappingLock(pVM, &Lock);
4927	#ifdef VBOX_VMM_TARGET_X86
4928	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
4929	#elif !defined(VBOX_VMM_TARGET_ARMV8)
4930	# error "misconfig"
4931	#endif
4932	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + GUEST_PAGE_SIZE);
4933	pvSrc = (const uint8_t *)pvSrc + GUEST_PAGE_SIZE;
4934	cb -= GUEST_PAGE_SIZE;
4935	}
4936	/* won't ever get here. */
4937	}
4938
4939
4940	/**
4941	* Read from guest physical memory referenced by GC pointer.
4942	*
4943	* This function uses the current CR3/CR0/CR4 of the guest and will
4944	* respect access handlers and set accessed bits.
4945	*
4946	* @returns Strict VBox status, see PGMPhysRead for details.
4947	* @retval VERR_PAGE_TABLE_NOT_PRESENT if there is no page mapped at the
4948	* specified virtual address.
4949	*
4950	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4951	* @param pvDst The destination address.
4952	* @param GCPtrSrc The source address (GC pointer).
4953	* @param cb The number of bytes to read.
4954	* @param enmOrigin Who is calling.
4955	* @thread EMT(pVCpu)
4956	*/
4957	VMMDECL(VBOXSTRICTRC) PGMPhysReadGCPtr(PVMCPUCC pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb, PGMACCESSORIGIN enmOrigin)
4958	{
4959	int rc;
4960	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4961	VMCPU_ASSERT_EMT(pVCpu);
4962
4963	/*
4964	* Anything to do?
4965	*/
4966	if (!cb)
4967	return VINF_SUCCESS;
4968
4969	LogFlow(("PGMPhysReadGCPtr: %RGv %zu\n", GCPtrSrc, cb));
4970
4971	/*
4972	* Optimize reads within a single page.
4973	*/
4974	if (((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK) + cb <= GUEST_PAGE_SIZE)
4975	{
4976	/* Convert virtual to physical address + flags */
4977	PGMPTWALK Walk;
4978	rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrSrc, &Walk);
4979	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
4980	RTGCPHYS const GCPhys = Walk.GCPhys \| ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
4981
4982	#ifdef VBOX_VMM_TARGET_X86
4983	/* mark the guest page as accessed. */
4984	if (!(Walk.fEffective & X86_PTE_A))
4985	{
4986	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
4987	AssertRC(rc);
4988	}
4989	#elif !defined(VBOX_VMM_TARGET_ARMV8)
4990	# error "misconfig"
4991	#endif
4992	return PGMPhysRead(pVM, GCPhys, pvDst, cb, enmOrigin);
4993	}
4994
4995	/*
4996	* Page by page.
4997	*/
4998	for (;;)
4999	{
5000	/* Convert virtual to physical address + flags */
5001	PGMPTWALK Walk;
5002	rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrSrc, &Walk);
5003	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
5004	RTGCPHYS const GCPhys = Walk.GCPhys \| ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
5005
5006	#ifdef VBOX_VMM_TARGET_X86
5007	/* mark the guest page as accessed. */
5008	if (!(Walk.fEffective & X86_PTE_A))
5009	{
5010	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
5011	AssertRC(rc);
5012	}
5013	#elif !defined(VBOX_VMM_TARGET_ARMV8)
5014	# error "misconfig"
5015	#endif
5016
5017	/* copy */
5018	size_t cbRead = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
5019	if (cbRead < cb)
5020	{
5021	VBOXSTRICTRC rcStrict = PGMPhysRead(pVM, GCPhys, pvDst, cbRead, enmOrigin);
5022	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
5023	{ /* likely */ }
5024	else
5025	return rcStrict;
5026	}
5027	else /* Last page (cbRead is GUEST_PAGE_SIZE, we only need cb!) */
5028	return PGMPhysRead(pVM, GCPhys, pvDst, cb, enmOrigin);
5029
5030	/* next */
5031	Assert(cb > cbRead);
5032	cb -= cbRead;
5033	pvDst = (uint8_t *)pvDst + cbRead;
5034	GCPtrSrc += cbRead;
5035	}
5036	}
5037
5038
5039	/**
5040	* Write to guest physical memory referenced by GC pointer.
5041	*
5042	* This function uses the current CR3/CR0/CR4 of the guest and will
5043	* respect access handlers and set dirty and accessed bits.
5044	*
5045	* @returns Strict VBox status, see PGMPhysWrite for details.
5046	* @retval VERR_PAGE_TABLE_NOT_PRESENT if there is no page mapped at the
5047	* specified virtual address.
5048	*
5049	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
5050	* @param GCPtrDst The destination address (GC pointer).
5051	* @param pvSrc The source address.
5052	* @param cb The number of bytes to write.
5053	* @param enmOrigin Who is calling.
5054	*/
5055	VMMDECL(VBOXSTRICTRC) PGMPhysWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb, PGMACCESSORIGIN enmOrigin)
5056	{
5057	int rc;
5058	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5059	VMCPU_ASSERT_EMT(pVCpu);
5060
5061	/*
5062	* Anything to do?
5063	*/
5064	if (!cb)
5065	return VINF_SUCCESS;
5066
5067	LogFlow(("PGMPhysWriteGCPtr: %RGv %zu\n", GCPtrDst, cb));
5068
5069	/*
5070	* Optimize writes within a single page.
5071	*/
5072	if (((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK) + cb <= GUEST_PAGE_SIZE)
5073	{
5074	/* Convert virtual to physical address + flags */
5075	PGMPTWALK Walk;
5076	rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrDst, &Walk);
5077	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
5078	RTGCPHYS const GCPhys = Walk.GCPhys \| ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
5079
5080	/* Mention when we ignore X86_PTE_RW... */
5081	if (!(Walk.fEffective & X86_PTE_RW))
5082	Log(("PGMPhysWriteGCPtr: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
5083
5084	#ifdef VBOX_VMM_TARGET_X86
5085	/* Mark the guest page as accessed and dirty if necessary. */
5086	if ((Walk.fEffective & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
5087	{
5088	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
5089	AssertRC(rc);
5090	}
5091	#elif !defined(VBOX_VMM_TARGET_ARMV8)
5092	# error "misconfig"
5093	#endif
5094
5095	return PGMPhysWrite(pVM, GCPhys, pvSrc, cb, enmOrigin);
5096	}
5097
5098	/*
5099	* Page by page.
5100	*/
5101	for (;;)
5102	{
5103	/* Convert virtual to physical address + flags */
5104	PGMPTWALK Walk;
5105	rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrDst, &Walk);
5106	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
5107	RTGCPHYS const GCPhys = Walk.GCPhys \| ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
5108
5109	/* Mention when we ignore X86_PTE_RW... */
5110	if (!(Walk.fEffective & X86_PTE_RW))
5111	Log(("PGMPhysWriteGCPtr: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
5112
5113	#ifdef VBOX_VMM_TARGET_X86
5114	/* Mark the guest page as accessed and dirty if necessary. */
5115	if ((Walk.fEffective & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
5116	{
5117	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
5118	AssertRC(rc);
5119	}
5120	#elif !defined(VBOX_VMM_TARGET_ARMV8)
5121	# error "misconfig"
5122	#endif
5123
5124	/* copy */
5125	size_t cbWrite = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
5126	if (cbWrite < cb)
5127	{
5128	VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM, GCPhys, pvSrc, cbWrite, enmOrigin);
5129	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
5130	{ /* likely */ }
5131	else
5132	return rcStrict;
5133	}
5134	else /* Last page (cbWrite is GUEST_PAGE_SIZE, we only need cb!) */
5135	return PGMPhysWrite(pVM, GCPhys, pvSrc, cb, enmOrigin);
5136
5137	/* next */
5138	Assert(cb > cbWrite);
5139	cb -= cbWrite;
5140	pvSrc = (uint8_t *)pvSrc + cbWrite;
5141	GCPtrDst += cbWrite;
5142	}
5143	}
5144
5145
5146	/**
5147	* Return the page type of the specified physical address.
5148	*
5149	* @returns The page type.
5150	* @param pVM The cross context VM structure.
5151	* @param GCPhys Guest physical address
5152	*/
5153	VMM_INT_DECL(PGMPAGETYPE) PGMPhysGetPageType(PVMCC pVM, RTGCPHYS GCPhys)
5154	{
5155	PGM_LOCK_VOID(pVM);
5156	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
5157	PGMPAGETYPE enmPgType = pPage ? (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage) : PGMPAGETYPE_INVALID;
5158	PGM_UNLOCK(pVM);
5159
5160	return enmPgType;
5161	}
5162
5163
5164	/** Helper for PGMPhysIemGCPhys2PtrNoLock. */
5165	DECL_FORCE_INLINE(int)
5166	pgmPhyIemGCphys2PtrNoLockReturnNoNothing(uint64_t uTlbPhysRev, R3R0PTRTYPE(uint8_t ) ppb, uint64_t *pfTlb,
5167	RTGCPHYS GCPhys, PCPGMPAGE pPageCopy)
5168	{
5169	*pfTlb \|= uTlbPhysRev
5170	\| PGMIEMGCPHYS2PTR_F_NO_WRITE \| PGMIEMGCPHYS2PTR_F_NO_READ \| PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
5171	*ppb = NULL;
5172	Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp ppb=NULL pfTlb=%#RX64 PageCopy=%R[pgmpage] NO\n", GCPhys,
5173	uTlbPhysRev \| PGMIEMGCPHYS2PTR_F_NO_WRITE \| PGMIEMGCPHYS2PTR_F_NO_READ \| PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3, pPageCopy));
5174	RT_NOREF(GCPhys, pPageCopy);
5175	return VINF_SUCCESS;
5176	}
5177
5178
5179	/** Helper for PGMPhysIemGCPhys2PtrNoLock. */
5180	DECL_FORCE_INLINE(int)
5181	pgmPhyIemGCphys2PtrNoLockReturnReadOnly(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uTlbPhysRev, RTGCPHYS GCPhys, PCPGMPAGE pPageCopy,
5182	PPGMRAMRANGE pRam, PPGMPAGE pPage, R3R0PTRTYPE(uint8_t ) ppb, uint64_t *pfTlb)
5183	{
5184	RT_NOREF(GCPhys);
5185	if (!PGM_PAGE_IS_CODE_PAGE(pPageCopy))
5186	*pfTlb \|= uTlbPhysRev \| PGMIEMGCPHYS2PTR_F_NO_WRITE;
5187	else
5188	*pfTlb \|= uTlbPhysRev \| PGMIEMGCPHYS2PTR_F_NO_WRITE \| PGMIEMGCPHYS2PTR_F_CODE_PAGE;
5189
5190	#ifdef IN_RING3
5191	if (PGM_IS_IN_NEM_MODE(pVM))
5192	*ppb = &pRam->pbR3[(RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << GUEST_PAGE_SHIFT];
5193	else
5194	#endif
5195	{
5196	#ifdef IN_RING3
5197	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
5198	PPGMPAGEMAPTLBE pTlbe;
5199	int rc = pgmPhysPageQueryLocklessTlbeWithPage(pVCpu, pPage, GCPhys, &pTlbe);
5200	AssertLogRelRCReturn(rc, rc);
5201	ppb = (uint8_t )pTlbe->pv;
5202	RT_NOREF(pVM);
5203	# endif
5204	#else /** @todo a safe lockless page TLB in ring-0 needs the to ensure it gets the right invalidations. later. */
5205	PGM_LOCK(pVM);
5206	PPGMPAGEMAPTLBE pTlbe;
5207	int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
5208	AssertLogRelRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
5209	ppb = (uint8_t )pTlbe->pv;
5210	PGM_UNLOCK(pVM);
5211	RT_NOREF(pVCpu);
5212	#endif
5213	}
5214	Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp ppb=%p pfTlb=%#RX64 PageCopy=%R[pgmpage] RO\n", GCPhys, ppb, pfTlb, pPageCopy));
5215	RT_NOREF(pRam, pVM, pVCpu);
5216	return VINF_SUCCESS;
5217	}
5218
5219
5220	/** Helper for PGMPhysIemGCPhys2PtrNoLock. */
5221	DECL_FORCE_INLINE(int)
5222	pgmPhyIemGCphys2PtrNoLockReturnReadWrite(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uTlbPhysRev, RTGCPHYS GCPhys, PCPGMPAGE pPageCopy,
5223	PPGMRAMRANGE pRam, PPGMPAGE pPage, R3R0PTRTYPE(uint8_t ) ppb, uint64_t *pfTlb)
5224	{
5225	Assert(!PGM_PAGE_IS_CODE_PAGE(pPageCopy));
5226	RT_NOREF(pPageCopy, GCPhys);
5227	*pfTlb \|= uTlbPhysRev;
5228
5229	#ifdef IN_RING3
5230	if (PGM_IS_IN_NEM_MODE(pVM))
5231	*ppb = &pRam->pbR3[(RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << GUEST_PAGE_SHIFT];
5232	else
5233	#endif
5234	{
5235	#ifdef IN_RING3
5236	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
5237	PPGMPAGEMAPTLBE pTlbe;
5238	int rc = pgmPhysPageQueryLocklessTlbeWithPage(pVCpu, pPage, GCPhys, &pTlbe);
5239	AssertLogRelRCReturn(rc, rc);
5240	ppb = (uint8_t )pTlbe->pv;
5241	RT_NOREF(pVM);
5242	# endif
5243	#else /** @todo a safe lockless page TLB in ring-0 needs the to ensure it gets the right invalidations. later. */
5244	PGM_LOCK(pVM);
5245	PPGMPAGEMAPTLBE pTlbe;
5246	int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
5247	AssertLogRelRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
5248	ppb = (uint8_t )pTlbe->pv;
5249	PGM_UNLOCK(pVM);
5250	RT_NOREF(pVCpu);
5251	#endif
5252	}
5253	Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp ppb=%p pfTlb=%#RX64 PageCopy=%R[pgmpage] RW\n", GCPhys, ppb, pfTlb, pPageCopy));
5254	RT_NOREF(pRam, pVM, pVCpu);
5255	return VINF_SUCCESS;
5256	}
5257
5258
5259	/**
5260	* Converts a GC physical address to a HC ring-3 pointer, with some
5261	* additional checks.
5262	*
5263	* @returns VBox status code (no informational statuses).
5264	*
5265	* @param pVM The cross context VM structure.
5266	* @param pVCpu The cross context virtual CPU structure of the
5267	* calling EMT.
5268	* @param GCPhys The GC physical address to convert. This API mask
5269	* the A20 line when necessary.
5270	* @param puTlbPhysRev Where to read the physical TLB revision. Needs to
5271	* be done while holding the PGM lock.
5272	* @param ppb Where to store the pointer corresponding to GCPhys
5273	* on success.
5274	* @param pfTlb The TLB flags and revision. We only add stuff.
5275	*
5276	* @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr and
5277	* PGMPhysIemGCPhys2Ptr.
5278	*
5279	* @thread EMT(pVCpu).
5280	*/
5281	VMM_INT_DECL(int) PGMPhysIemGCPhys2PtrNoLock(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint64_t const volatile *puTlbPhysRev,
5282	R3R0PTRTYPE(uint8_t ) ppb, uint64_t *pfTlb)
5283	{
5284	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
5285	Assert(!(GCPhys & X86_PAGE_OFFSET_MASK));
5286
5287	PGMRAMRANGE volatile *pRam;
5288	PGMPAGE volatile *pPage;
5289	int rc = pgmPhysGetPageAndRangeExLockless(pVM, pVCpu, GCPhys, &pPage, &pRam);
5290	if (RT_SUCCESS(rc))
5291	{
5292	/*
5293	* Wrt to update races, we will try to pretend we beat the update we're
5294	* racing. We do this by sampling the physical TLB revision first, so
5295	* that the TLB entry / whatever purpose the caller has with the info
5296	* will become invalid immediately if it's updated.
5297	*
5298	* This means the caller will (probably) make use of the returned info
5299	* only once and then requery it the next time it is use, getting the
5300	* updated info. This would then be just as if the first query got the
5301	* PGM lock before the updater.
5302	*/
5303	/** @todo make PGMPAGE updates more atomic, possibly flagging complex
5304	* updates by adding a u1UpdateInProgress field (or revision).
5305	* This would be especially important when updating the page ID... */
5306	uint64_t uTlbPhysRev = *puTlbPhysRev;
5307	PGMPAGE PageCopy = { { pPage->au64[0], pPage->au64[1] } };
5308	if ( uTlbPhysRev == *puTlbPhysRev
5309	&& PageCopy.au64[0] == pPage->au64[0]
5310	&& PageCopy.au64[1] == pPage->au64[1])
5311	ASMCompilerBarrier(); /* likely */
5312	else
5313	{
5314	PGM_LOCK_VOID(pVM);
5315	uTlbPhysRev = *puTlbPhysRev;
5316	PageCopy.au64[0] = pPage->au64[0];
5317	PageCopy.au64[1] = pPage->au64[1];
5318	PGM_UNLOCK(pVM);
5319	}
5320
5321	/*
5322	* Try optimize for the regular case first: Writable RAM.
5323	*/
5324	switch (PGM_PAGE_GET_HNDL_PHYS_STATE(&PageCopy))
5325	{
5326	case PGM_PAGE_HNDL_PHYS_STATE_DISABLED:
5327	if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(&PageCopy))
5328	{ /* likely */ }
5329	else
5330	return pgmPhyIemGCphys2PtrNoLockReturnNoNothing(uTlbPhysRev, ppb, pfTlb, GCPhys, &PageCopy);
5331	RT_FALL_THRU();
5332	case PGM_PAGE_HNDL_PHYS_STATE_NONE:
5333	Assert(!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(&PageCopy));
5334	switch (PGM_PAGE_GET_STATE_NA(&PageCopy))
5335	{
5336	case PGM_PAGE_STATE_ALLOCATED:
5337	return pgmPhyIemGCphys2PtrNoLockReturnReadWrite(pVM, pVCpu, uTlbPhysRev, GCPhys, &PageCopy,
5338	(PPGMRAMRANGE)pRam, (PPGMPAGE)pPage, ppb, pfTlb);
5339
5340	case PGM_PAGE_STATE_ZERO:
5341	case PGM_PAGE_STATE_WRITE_MONITORED:
5342	case PGM_PAGE_STATE_SHARED:
5343	return pgmPhyIemGCphys2PtrNoLockReturnReadOnly(pVM, pVCpu, uTlbPhysRev, GCPhys, &PageCopy,
5344	(PPGMRAMRANGE)pRam, (PPGMPAGE)pPage, ppb, pfTlb);
5345
5346	default: AssertFailed(); RT_FALL_THROUGH();
5347	case PGM_PAGE_STATE_BALLOONED:
5348	return pgmPhyIemGCphys2PtrNoLockReturnNoNothing(uTlbPhysRev, ppb, pfTlb, GCPhys, &PageCopy);
5349	}
5350	break;
5351
5352	case PGM_PAGE_HNDL_PHYS_STATE_WRITE:
5353	Assert(!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(&PageCopy));
5354	switch (PGM_PAGE_GET_STATE_NA(&PageCopy))
5355	{
5356	case PGM_PAGE_STATE_ALLOCATED:
5357	Assert(!PGM_PAGE_IS_CODE_PAGE(&PageCopy));
5358	RT_FALL_THRU();
5359	case PGM_PAGE_STATE_ZERO:
5360	case PGM_PAGE_STATE_WRITE_MONITORED:
5361	case PGM_PAGE_STATE_SHARED:
5362	return pgmPhyIemGCphys2PtrNoLockReturnReadOnly(pVM, pVCpu, uTlbPhysRev, GCPhys, &PageCopy,
5363	(PPGMRAMRANGE)pRam, (PPGMPAGE)pPage, ppb, pfTlb);
5364
5365	default: AssertFailed(); RT_FALL_THROUGH();
5366	case PGM_PAGE_STATE_BALLOONED:
5367	return pgmPhyIemGCphys2PtrNoLockReturnNoNothing(uTlbPhysRev, ppb, pfTlb, GCPhys, &PageCopy);
5368	}
5369	break;
5370
5371	case PGM_PAGE_HNDL_PHYS_STATE_ALL:
5372	Assert(!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(&PageCopy));
5373	return pgmPhyIemGCphys2PtrNoLockReturnNoNothing(uTlbPhysRev, ppb, pfTlb, GCPhys, &PageCopy);
5374	}
5375	}
5376	else
5377	{
5378	pfTlb \|= puTlbPhysRev \| PGMIEMGCPHYS2PTR_F_NO_WRITE \| PGMIEMGCPHYS2PTR_F_NO_READ
5379	\| PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 \| PGMIEMGCPHYS2PTR_F_UNASSIGNED;
5380	*ppb = NULL;
5381	Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp ppb=%p pfTlb=%#RX64 (rc=%Rrc)\n", GCPhys, ppb, pfTlb, rc));
5382	}
5383
5384	return VINF_SUCCESS;
5385	}
5386
5387
5388	/**
5389	* Converts a GC physical address to a HC ring-3 pointer, with some
5390	* additional checks.
5391	*
5392	* @returns VBox status code (no informational statuses).
5393	* @retval VINF_SUCCESS on success.
5394	* @retval VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
5395	* access handler of some kind.
5396	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
5397	* accesses or is odd in any way.
5398	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
5399	*
5400	* @param pVM The cross context VM structure.
5401	* @param pVCpu The cross context virtual CPU structure of the
5402	* calling EMT.
5403	* @param GCPhys The GC physical address to convert. This API mask
5404	* the A20 line when necessary.
5405	* @param fWritable Whether write access is required.
5406	* @param fByPassHandlers Whether to bypass access handlers.
5407	* @param ppv Where to store the pointer corresponding to GCPhys
5408	* on success.
5409	* @param pLock
5410	*
5411	* @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr.
5412	* @thread EMT(pVCpu).
5413	*/
5414	VMM_INT_DECL(int) PGMPhysIemGCPhys2Ptr(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers,
5415	void **ppv, PPGMPAGEMAPLOCK pLock)
5416	{
5417	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
5418	RT_NOREF(pVCpu);
5419
5420	PGM_LOCK_VOID(pVM);
5421
5422	PPGMRAMRANGE pRam;
5423	PPGMPAGE pPage;
5424	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
5425	if (RT_SUCCESS(rc))
5426	{
5427	if (PGM_PAGE_IS_BALLOONED(pPage))
5428	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
5429	else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
5430	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
5431	else if ( !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
5432	\|\| (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
5433	rc = VINF_SUCCESS;
5434	else
5435	{
5436	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
5437	{
5438	Assert(!fByPassHandlers \|\| PGM_PAGE_IS_MMIO(pPage));
5439	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
5440	}
5441	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
5442	{
5443	Assert(!fByPassHandlers);
5444	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
5445	}
5446	}
5447	if (RT_SUCCESS(rc))
5448	{
5449	int rc2;
5450
5451	/* Make sure what we return is writable. */
5452	if (fWritable)
5453	switch (PGM_PAGE_GET_STATE(pPage))
5454	{
5455	case PGM_PAGE_STATE_ALLOCATED:
5456	break;
5457	case PGM_PAGE_STATE_BALLOONED:
5458	AssertFailed();
5459	break;
5460	case PGM_PAGE_STATE_ZERO:
5461	case PGM_PAGE_STATE_SHARED:
5462	case PGM_PAGE_STATE_WRITE_MONITORED:
5463	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
5464	AssertLogRelRCReturn(rc2, rc2);
5465	break;
5466	}
5467
5468	/* Get a ring-3 mapping of the address. */
5469	PPGMPAGEMAPTLBE pTlbe;
5470	rc2 = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
5471	AssertLogRelRCReturn(rc2, rc2);
5472
5473	/* Lock it and calculate the address. */
5474	if (fWritable)
5475	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
5476	else
5477	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
5478	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
5479
5480	Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
5481	}
5482	else
5483	Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
5484
5485	/* else: handler catching all access, no pointer returned. */
5486	}
5487	else
5488	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
5489
5490	PGM_UNLOCK(pVM);
5491	return rc;
5492	}
5493
5494
5495	/**
5496	* Checks if the give GCPhys page requires special handling for the given access
5497	* because it's MMIO or otherwise monitored.
5498	*
5499	* @returns VBox status code (no informational statuses).
5500	* @retval VINF_SUCCESS on success.
5501	* @retval VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
5502	* access handler of some kind.
5503	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
5504	* accesses or is odd in any way.
5505	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
5506	*
5507	* @param pVM The cross context VM structure.
5508	* @param GCPhys The GC physical address to convert. Since this is
5509	* only used for filling the REM TLB, the A20 mask must
5510	* be applied before calling this API.
5511	* @param fWritable Whether write access is required.
5512	* @param fByPassHandlers Whether to bypass access handlers.
5513	*
5514	* @remarks This is a watered down version PGMPhysIemGCPhys2Ptr and really just
5515	* a stop gap thing that should be removed once there is a better TLB
5516	* for virtual address accesses.
5517	*/
5518	VMM_INT_DECL(int) PGMPhysIemQueryAccess(PVMCC pVM, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers)
5519	{
5520	PGM_LOCK_VOID(pVM);
5521	PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
5522
5523	PPGMRAMRANGE pRam;
5524	PPGMPAGE pPage;
5525	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
5526	if (RT_SUCCESS(rc))
5527	{
5528	if (PGM_PAGE_IS_BALLOONED(pPage))
5529	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
5530	else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
5531	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
5532	else if ( !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
5533	\|\| (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
5534	rc = VINF_SUCCESS;
5535	else
5536	{
5537	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
5538	{
5539	Assert(!fByPassHandlers \|\| PGM_PAGE_IS_MMIO(pPage));
5540	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
5541	}
5542	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
5543	{
5544	Assert(!fByPassHandlers);
5545	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
5546	}
5547	}
5548	}
5549
5550	PGM_UNLOCK(pVM);
5551	return rc;
5552	}
5553
5554	#ifdef VBOX_WITH_NATIVE_NEM
5555
5556	/**
5557	* Interface used by NEM to check what to do on a memory access exit.
5558	*
5559	* @returns VBox status code.
5560	* @param pVM The cross context VM structure.
5561	* @param pVCpu The cross context per virtual CPU structure.
5562	* Optional.
5563	* @param GCPhys The guest physical address.
5564	* @param fMakeWritable Whether to try make the page writable or not. If it
5565	* cannot be made writable, NEM_PAGE_PROT_WRITE won't
5566	* be returned and the return code will be unaffected
5567	* @param pInfo Where to return the page information. This is
5568	* initialized even on failure.
5569	* @param pfnChecker Page in-sync checker callback. Optional.
5570	* @param pvUser User argument to pass to pfnChecker.
5571	*/
5572	VMM_INT_DECL(int) PGMPhysNemPageInfoChecker(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, bool fMakeWritable,
5573	PPGMPHYSNEMPAGEINFO pInfo, PFNPGMPHYSNEMCHECKPAGE pfnChecker, void *pvUser)
5574	{
5575	PGM_LOCK_VOID(pVM);
5576
5577	PPGMPAGE pPage;
5578	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
5579	if (RT_SUCCESS(rc))
5580	{
5581	/* Try make it writable if requested. */
5582	pInfo->u2OldNemState = PGM_PAGE_GET_NEM_STATE(pPage);
5583	if (fMakeWritable)
5584	switch (PGM_PAGE_GET_STATE(pPage))
5585	{
5586	case PGM_PAGE_STATE_SHARED:
5587	case PGM_PAGE_STATE_WRITE_MONITORED:
5588	case PGM_PAGE_STATE_ZERO:
5589	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
5590	if (rc == VERR_PGM_PHYS_PAGE_RESERVED)
5591	rc = VINF_SUCCESS;
5592	break;
5593	}
5594
5595	/* Fill in the info. */
5596	pInfo->HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
5597	pInfo->u2NemState = PGM_PAGE_GET_NEM_STATE(pPage);
5598	pInfo->fHasHandlers = PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) ? 1 : 0;
5599	PGMPAGETYPE const enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
5600	pInfo->enmType = enmType;
5601	pInfo->fNemProt = pgmPhysPageCalcNemProtection(pPage, enmType);
5602	switch (PGM_PAGE_GET_STATE(pPage))
5603	{
5604	case PGM_PAGE_STATE_ALLOCATED:
5605	pInfo->fZeroPage = 0;
5606	break;
5607
5608	case PGM_PAGE_STATE_ZERO:
5609	pInfo->fZeroPage = 1;
5610	break;
5611
5612	case PGM_PAGE_STATE_WRITE_MONITORED:
5613	pInfo->fZeroPage = 0;
5614	break;
5615
5616	case PGM_PAGE_STATE_SHARED:
5617	pInfo->fZeroPage = 0;
5618	break;
5619
5620	case PGM_PAGE_STATE_BALLOONED:
5621	pInfo->fZeroPage = 1;
5622	break;
5623
5624	default:
5625	pInfo->fZeroPage = 1;
5626	AssertFailedStmt(rc = VERR_PGM_PHYS_PAGE_GET_IPE);
5627	}
5628
5629	/* Call the checker and update NEM state. */
5630	if (pfnChecker)
5631	{
5632	rc = pfnChecker(pVM, pVCpu, GCPhys, pInfo, pvUser);
5633	PGM_PAGE_SET_NEM_STATE(pPage, pInfo->u2NemState);
5634	}
5635
5636	/* Done. */
5637	PGM_UNLOCK(pVM);
5638	}
5639	else
5640	{
5641	PGM_UNLOCK(pVM);
5642
5643	pInfo->HCPhys = NIL_RTHCPHYS;
5644	pInfo->fNemProt = NEM_PAGE_PROT_NONE;
5645	pInfo->u2NemState = 0;
5646	pInfo->fHasHandlers = 0;
5647	pInfo->fZeroPage = 0;
5648	pInfo->enmType = PGMPAGETYPE_INVALID;
5649	}
5650
5651	return rc;
5652	}
5653
5654
5655	/**
5656	* NEM helper that performs @a pfnCallback on pages with NEM state @a uMinState
5657	* or higher.
5658	*
5659	* @returns VBox status code from callback.
5660	* @param pVM The cross context VM structure.
5661	* @param pVCpu The cross context per CPU structure. This is
5662	* optional as its only for passing to callback.
5663	* @param uMinState The minimum NEM state value to call on.
5664	* @param pfnCallback The callback function.
5665	* @param pvUser User argument for the callback.
5666	*/
5667	VMM_INT_DECL(int) PGMPhysNemEnumPagesByState(PVMCC pVM, PVMCPUCC pVCpu, uint8_t uMinState,
5668	PFNPGMPHYSNEMENUMCALLBACK pfnCallback, void *pvUser)
5669	{
5670	/*
5671	* Just brute force this problem.
5672	*/
5673	PGM_LOCK_VOID(pVM);
5674	int rc = VINF_SUCCESS;
5675	uint32_t const cLookupEntries = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
5676	for (uint32_t idxLookup = 0; idxLookup < cLookupEntries && RT_SUCCESS(rc); idxLookup++)
5677	{
5678	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
5679	AssertContinue(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges));
5680	PPGMRAMRANGE const pRam = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
5681	AssertContinue(pRam);
5682	Assert(pRam->GCPhys == PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]));
5683
5684	#ifdef IN_RING0
5685	uint32_t const cPages = RT_MIN(pRam->cb >> X86_PAGE_SHIFT, pVM->pgmr0.s.acRamRangePages[idRamRange]);
5686	#else
5687	uint32_t const cPages = pRam->cb >> X86_PAGE_SHIFT;
5688	#endif
5689	for (uint32_t iPage = 0; iPage < cPages; iPage++)
5690	{
5691	uint8_t u2State = PGM_PAGE_GET_NEM_STATE(&pRam->aPages[iPage]);
5692	if (u2State < uMinState)
5693	{ /* likely */ }
5694	else
5695	{
5696	rc = pfnCallback(pVM, pVCpu, pRam->GCPhys + ((RTGCPHYS)iPage << X86_PAGE_SHIFT), &u2State, pvUser);
5697	if (RT_SUCCESS(rc))
5698	PGM_PAGE_SET_NEM_STATE(&pRam->aPages[iPage], u2State);
5699	else
5700	break;
5701	}
5702	}
5703	}
5704	PGM_UNLOCK(pVM);
5705
5706	return rc;
5707	}
5708
5709
5710	/**
5711	* Helper for setting the NEM state for a range of pages.
5712	*
5713	* @param paPages Array of pages to modify.
5714	* @param cPages How many pages to modify.
5715	* @param u2State The new state value.
5716	*/
5717	DECLHIDDEN(void) pgmPhysSetNemStateForPages(PPGMPAGE paPages, RTGCPHYS cPages, uint8_t u2State)
5718	{
5719	PPGMPAGE pPage = paPages;
5720	while (cPages-- > 0)
5721	{
5722	PGM_PAGE_SET_NEM_STATE(pPage, u2State);
5723	pPage++;
5724	}
5725	}
5726
5727	#endif /* VBOX_WITH_NATIVE_NEM */
5728

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp@ 108732

Download in other formats: