PGMAllPhys.cpp@ 107631

Last change on this file since 107631 was 107194, checked in by vboxsync, 7 weeks ago
VMM: More adjustments for VBOX_WITH_ONLY_PGM_NEM_MODE, VBOX_WITH_MINIMAL_R0, VBOX_WITH_HWVIRT and such. jiraref:VBP-1466
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 211.2 KB

Line
1	/* $Id: PGMAllPhys.cpp 107194 2024-11-29 14:47:06Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_PGM_PHYS
33	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
34	#include <VBox/vmm/pgm.h>
35	#include <VBox/vmm/trpm.h>
36	#include <VBox/vmm/vmm.h>
37	#include <VBox/vmm/iem.h>
38	#include <VBox/vmm/iom.h>
39	#include <VBox/vmm/em.h>
40	#include <VBox/vmm/nem.h>
41	#include "PGMInternal.h"
42	#include <VBox/vmm/vmcc.h>
43	#include "PGMInline.h"
44	#include <VBox/param.h>
45	#include <VBox/err.h>
46	#include <iprt/assert.h>
47	#include <iprt/string.h>
48	#include <VBox/log.h>
49	#ifdef IN_RING3
50	# include <iprt/thread.h>
51	#elif defined(IN_RING0)
52	# include <iprt/mem.h>
53	# include <iprt/memobj.h>
54	#endif
55
56
57	/*********************************************************************************************************************************
58	* Defined Constants And Macros *
59	*********************************************************************************************************************************/
60	/** Enable the physical TLB. */
61	#define PGM_WITH_PHYS_TLB
62
63	/** @def PGM_HANDLER_PHYS_IS_VALID_STATUS
64	* Checks if valid physical access handler return code (normal handler, not PF).
65	*
66	* Checks if the given strict status code is one of the expected ones for a
67	* physical access handler in the current context.
68	*
69	* @returns true or false.
70	* @param a_rcStrict The status code.
71	* @param a_fWrite Whether it is a write or read being serviced.
72	*
73	* @remarks We wish to keep the list of statuses here as short as possible.
74	* When changing, please make sure to update the PGMPhysRead,
75	* PGMPhysWrite, PGMPhysReadGCPtr and PGMPhysWriteGCPtr docs too.
76	*/
77	#ifdef IN_RING3
78	# define PGM_HANDLER_PHYS_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
79	( (a_rcStrict) == VINF_SUCCESS \
80	\|\| (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT)
81	#elif defined(IN_RING0)
82	#define PGM_HANDLER_PHYS_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
83	( (a_rcStrict) == VINF_SUCCESS \
84	\|\| (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT \
85	\
86	\|\| (a_rcStrict) == ((a_fWrite) ? VINF_IOM_R3_MMIO_WRITE : VINF_IOM_R3_MMIO_READ) \
87	\|\| (a_rcStrict) == VINF_IOM_R3_MMIO_READ_WRITE \
88	\|\| ((a_rcStrict) == VINF_IOM_R3_MMIO_COMMIT_WRITE && (a_fWrite)) \
89	\
90	\|\| (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR \
91	\|\| (a_rcStrict) == VINF_EM_DBG_STOP \
92	\|\| (a_rcStrict) == VINF_EM_DBG_EVENT \
93	\|\| (a_rcStrict) == VINF_EM_DBG_BREAKPOINT \
94	\|\| (a_rcStrict) == VINF_EM_OFF \
95	\|\| (a_rcStrict) == VINF_EM_SUSPEND \
96	\|\| (a_rcStrict) == VINF_EM_RESET \
97	)
98	#else
99	# error "Context?"
100	#endif
101
102	/** @def PGM_HANDLER_VIRT_IS_VALID_STATUS
103	* Checks if valid virtual access handler return code (normal handler, not PF).
104	*
105	* Checks if the given strict status code is one of the expected ones for a
106	* virtual access handler in the current context.
107	*
108	* @returns true or false.
109	* @param a_rcStrict The status code.
110	* @param a_fWrite Whether it is a write or read being serviced.
111	*
112	* @remarks We wish to keep the list of statuses here as short as possible.
113	* When changing, please make sure to update the PGMPhysRead,
114	* PGMPhysWrite, PGMPhysReadGCPtr and PGMPhysWriteGCPtr docs too.
115	*/
116	#ifdef IN_RING3
117	# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
118	( (a_rcStrict) == VINF_SUCCESS \
119	\|\| (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT)
120	#elif defined(IN_RING0)
121	# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
122	(false /* no virtual handlers in ring-0! */ )
123	#else
124	# error "Context?"
125	#endif
126
127
128
129	/**
130	* Calculate the actual table size.
131	*
132	* The memory is layed out like this:
133	* - PGMPHYSHANDLERTREE (8 bytes)
134	* - Allocation bitmap (8-byte size align)
135	* - Slab of PGMPHYSHANDLER. Start is 64 byte aligned.
136	*/
137	uint32_t pgmHandlerPhysicalCalcTableSizes(uint32_t pcEntries, uint32_t pcbTreeAndBitmap)
138	{
139	/*
140	* A minimum of 64 entries and a maximum of ~64K.
141	*/
142	uint32_t cEntries = *pcEntries;
143	if (cEntries <= 64)
144	cEntries = 64;
145	else if (cEntries >= _64K)
146	cEntries = _64K;
147	else
148	cEntries = RT_ALIGN_32(cEntries, 16);
149
150	/*
151	* Do the initial calculation.
152	*/
153	uint32_t cbBitmap = RT_ALIGN_32(cEntries, 64) / 8;
154	uint32_t cbTreeAndBitmap = RT_ALIGN_32(sizeof(PGMPHYSHANDLERTREE) + cbBitmap, 64);
155	uint32_t cbTable = cEntries * sizeof(PGMPHYSHANDLER);
156	uint32_t cbTotal = cbTreeAndBitmap + cbTable;
157
158	/*
159	* Align the total and try use up extra space from that.
160	*/
161	uint32_t cbTotalAligned = RT_ALIGN_32(cbTotal, RT_MAX(HOST_PAGE_SIZE, _16K));
162	uint32_t cAvail = cbTotalAligned - cbTotal;
163	cAvail /= sizeof(PGMPHYSHANDLER);
164	if (cAvail >= 1)
165	for (;;)
166	{
167	cbBitmap = RT_ALIGN_32(cEntries, 64) / 8;
168	cbTreeAndBitmap = RT_ALIGN_32(sizeof(PGMPHYSHANDLERTREE) + cbBitmap, 64);
169	cbTable = cEntries * sizeof(PGMPHYSHANDLER);
170	cbTotal = cbTreeAndBitmap + cbTable;
171	if (cbTotal <= cbTotalAligned)
172	break;
173	cEntries--;
174	Assert(cEntries >= 16);
175	}
176
177	/*
178	* Return the result.
179	*/
180	*pcbTreeAndBitmap = cbTreeAndBitmap;
181	*pcEntries = cEntries;
182	return cbTotalAligned;
183	}
184
185
186
187	/*********************************************************************************************************************************
188	* Access Handlers for ROM and MMIO2 *
189	*********************************************************************************************************************************/
190
191	#ifndef IN_RING3
192
193	/**
194	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
195	* \#PF access handler callback for guest ROM range write access.}
196	*
197	* @remarks The @a uUser argument is the PGMROMRANGE::GCPhys value.
198	*/
199	DECLCALLBACK(VBOXSTRICTRC) pgmPhysRomWritePfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx,
200	RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
201
202	{
203	AssertReturn(uUser < RT_ELEMENTS(pVM->pgmr0.s.apRomRanges), VINF_EM_RAW_EMULATE_INSTR);
204	PPGMROMRANGE const pRom = pVM->pgmr0.s.apRomRanges[uUser];
205	AssertReturn(pRom, VINF_EM_RAW_EMULATE_INSTR);
206
207	uint32_t const iPage = (GCPhysFault - pRom->GCPhys) >> GUEST_PAGE_SHIFT;
208	AssertReturn(iPage < (pRom->cb >> GUEST_PAGE_SHIFT), VERR_INTERNAL_ERROR_3);
209	#ifdef IN_RING0
210	AssertReturn(iPage < pVM->pgmr0.s.acRomRangePages[uUser], VERR_INTERNAL_ERROR_2);
211	#endif
212
213	RT_NOREF(uErrorCode, pvFault);
214	Assert(uErrorCode & X86_TRAP_PF_RW); /* This shall not be used for read access! */
215
216	int rc;
217	switch (pRom->aPages[iPage].enmProt)
218	{
219	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
220	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
221	{
222	/*
223	* If it's a simple instruction which doesn't change the cpu state
224	* we will simply skip it. Otherwise we'll have to defer it to REM.
225	*/
226	uint32_t cbOp;
227	PDISSTATE pDis = &pVCpu->pgm.s.Dis;
228	rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbOp);
229	if ( RT_SUCCESS(rc)
230	&& pDis->uCpuMode == DISCPUMODE_32BIT /** @todo why does this matter? */
231	&& !(pDis->x86.fPrefix & (DISPREFIX_REPNE \| DISPREFIX_REP \| DISPREFIX_SEG)))
232	{
233	switch (pDis->x86.bOpCode)
234	{
235	/** @todo Find other instructions we can safely skip, possibly
236	* adding this kind of detection to DIS or EM. */
237	case OP_MOV:
238	pCtx->rip += cbOp;
239	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZGuestROMWriteHandled);
240	return VINF_SUCCESS;
241	}
242	}
243	break;
244	}
245
246	case PGMROMPROT_READ_RAM_WRITE_RAM:
247	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
248	rc = PGMHandlerPhysicalPageTempOff(pVM, pRom->GCPhys, GCPhysFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
249	AssertRC(rc);
250	break; /** @todo Must edit the shadow PT and restart the instruction, not use the interpreter! */
251
252	case PGMROMPROT_READ_ROM_WRITE_RAM:
253	/* Handle it in ring-3 because it's way easier there. */
254	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
255	break;
256
257	default:
258	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhysFault=%RGp\n",
259	pRom->aPages[iPage].enmProt, iPage, GCPhysFault),
260	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
261	}
262
263	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZGuestROMWriteUnhandled);
264	return VINF_EM_RAW_EMULATE_INSTR;
265	}
266
267	#endif /* !IN_RING3 */
268
269
270	/**
271	* @callback_method_impl{FNPGMPHYSHANDLER,
272	* Access handler callback for ROM write accesses.}
273	*
274	* @remarks The @a uUser argument is the PGMROMRANGE::GCPhys value.
275	*/
276	DECLCALLBACK(VBOXSTRICTRC)
277	pgmPhysRomWriteHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
278	PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser)
279	{
280	AssertReturn(uUser < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRomRanges), VERR_INTERNAL_ERROR_3);
281	PPGMROMRANGE const pRom = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRomRanges[uUser];
282	AssertReturn(pRom, VERR_INTERNAL_ERROR_3);
283
284	uint32_t const iPage = (GCPhys - pRom->GCPhys) >> GUEST_PAGE_SHIFT;
285	AssertReturn(iPage < (pRom->cb >> GUEST_PAGE_SHIFT), VERR_INTERNAL_ERROR_2);
286	#ifdef IN_RING0
287	AssertReturn(iPage < pVM->pgmr0.s.acRomRangePages[uUser], VERR_INTERNAL_ERROR_2);
288	#endif
289	PPGMROMPAGE const pRomPage = &pRom->aPages[iPage];
290
291	Log5(("pgmPhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
292	RT_NOREF(pVCpu, pvPhys, enmOrigin);
293
294	if (enmAccessType == PGMACCESSTYPE_READ)
295	{
296	switch (pRomPage->enmProt)
297	{
298	/*
299	* Take the default action.
300	*/
301	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
302	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
303	case PGMROMPROT_READ_ROM_WRITE_RAM:
304	case PGMROMPROT_READ_RAM_WRITE_RAM:
305	return VINF_PGM_HANDLER_DO_DEFAULT;
306
307	default:
308	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
309	pRom->aPages[iPage].enmProt, iPage, GCPhys),
310	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
311	}
312	}
313	else
314	{
315	Assert(enmAccessType == PGMACCESSTYPE_WRITE);
316	switch (pRomPage->enmProt)
317	{
318	/*
319	* Ignore writes.
320	*/
321	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
322	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
323	return VINF_SUCCESS;
324
325	/*
326	* Write to the RAM page.
327	*/
328	case PGMROMPROT_READ_ROM_WRITE_RAM:
329	case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's way simpler that way. */
330	{
331	/* This should be impossible now, pvPhys doesn't work cross page anylonger. */
332	Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> GUEST_PAGE_SHIFT) == iPage);
333
334	/*
335	* Take the lock, do lazy allocation, map the page and copy the data.
336	*
337	* Note that we have to bypass the mapping TLB since it works on
338	* guest physical addresses and entering the shadow page would
339	* kind of screw things up...
340	*/
341	PGM_LOCK_VOID(pVM);
342
343	PPGMPAGE pShadowPage = &pRomPage->Shadow;
344	if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
345	{
346	pShadowPage = pgmPhysGetPage(pVM, GCPhys);
347	AssertLogRelMsgReturnStmt(pShadowPage, ("%RGp\n", GCPhys), PGM_UNLOCK(pVM), VERR_PGM_PHYS_PAGE_GET_IPE);
348	}
349
350	void *pvDstPage;
351	int rc;
352	#if defined(VBOX_WITH_PGM_NEM_MODE) && defined(IN_RING3)
353	if (PGM_IS_IN_NEM_MODE(pVM) && PGMROMPROT_IS_ROM(pRomPage->enmProt))
354	{
355	pvDstPage = &pRom->pbR3Alternate[GCPhys - pRom->GCPhys];
356	rc = VINF_SUCCESS;
357	}
358	else
359	#endif
360	{
361	rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK, &pvDstPage);
362	if (RT_SUCCESS(rc))
363	pvDstPage = (uint8_t *)pvDstPage + (GCPhys & GUEST_PAGE_OFFSET_MASK);
364	}
365	if (RT_SUCCESS(rc))
366	{
367	memcpy((uint8_t *)pvDstPage + (GCPhys & GUEST_PAGE_OFFSET_MASK), pvBuf, cbBuf);
368	pRomPage->LiveSave.fWrittenTo = true;
369
370	AssertMsg( rc == VINF_SUCCESS
371	\|\| ( rc == VINF_PGM_SYNC_CR3
372	&& VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
373	, ("%Rrc\n", rc));
374	rc = VINF_SUCCESS;
375	}
376
377	PGM_UNLOCK(pVM);
378	return rc;
379	}
380
381	default:
382	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
383	pRom->aPages[iPage].enmProt, iPage, GCPhys),
384	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
385	}
386	}
387	}
388
389
390	/**
391	* Common worker for pgmPhysMmio2WriteHandler and pgmPhysMmio2WritePfHandler.
392	*/
393	static VBOXSTRICTRC pgmPhysMmio2WriteHandlerCommon(PVMCC pVM, PVMCPUCC pVCpu, uint64_t hMmio2, RTGCPHYS GCPhys, RTGCPTR GCPtr)
394	{
395	/*
396	* Get the MMIO2 range.
397	*/
398	AssertReturn(hMmio2 < RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges), VERR_INTERNAL_ERROR_3);
399	AssertReturn(hMmio2 != 0, VERR_INTERNAL_ERROR_3);
400	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[hMmio2 - 1];
401	Assert(pMmio2->idMmio2 == hMmio2);
402	AssertReturn((pMmio2->fFlags & PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES) == PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES,
403	VERR_INTERNAL_ERROR_4);
404
405	/*
406	* Get the page and make sure it's an MMIO2 page.
407	*/
408	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
409	AssertReturn(pPage, VINF_EM_RAW_EMULATE_INSTR);
410	AssertReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2, VINF_EM_RAW_EMULATE_INSTR);
411
412	/*
413	* Set the dirty flag so we can avoid scanning all the pages when it isn't dirty.
414	* (The PGM_PAGE_HNDL_PHYS_STATE_DISABLED handler state indicates that a single
415	* page is dirty, saving the need for additional storage (bitmap).)
416	*/
417	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_IS_DIRTY;
418
419	/*
420	* Disable the handler for this page.
421	*/
422	int rc = PGMHandlerPhysicalPageTempOff(pVM, pMmio2->GCPhys, GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
423	AssertRC(rc);
424	#ifndef IN_RING3
425	if (RT_SUCCESS(rc) && GCPtr != ~(RTGCPTR)0)
426	{
427	rc = PGMShwMakePageWritable(pVCpu, GCPtr, PGM_MK_PG_IS_MMIO2 \| PGM_MK_PG_IS_WRITE_FAULT);
428	AssertMsgReturn(rc == VINF_SUCCESS \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT,
429	("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", GCPtr, rc), rc);
430	}
431	#else
432	RT_NOREF(pVCpu, GCPtr);
433	#endif
434	return VINF_SUCCESS;
435	}
436
437
438	#ifndef IN_RING3
439	/**
440	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
441	* \#PF access handler callback for guest MMIO2 dirty page tracing.}
442	*
443	* @remarks The @a uUser is the MMIO2 index.
444	*/
445	DECLCALLBACK(VBOXSTRICTRC) pgmPhysMmio2WritePfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx,
446	RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
447	{
448	RT_NOREF(pVCpu, uErrorCode, pCtx);
449	VBOXSTRICTRC rcStrict = PGM_LOCK(pVM); /* We should already have it, but just make sure we do. */
450	if (RT_SUCCESS(rcStrict))
451	{
452	rcStrict = pgmPhysMmio2WriteHandlerCommon(pVM, pVCpu, uUser, GCPhysFault, pvFault);
453	PGM_UNLOCK(pVM);
454	}
455	return rcStrict;
456	}
457	#endif /* !IN_RING3 */
458
459
460	/**
461	* @callback_method_impl{FNPGMPHYSHANDLER,
462	* Access handler callback for MMIO2 dirty page tracing.}
463	*
464	* @remarks The @a uUser is the MMIO2 index.
465	*/
466	DECLCALLBACK(VBOXSTRICTRC)
467	pgmPhysMmio2WriteHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
468	PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser)
469	{
470	VBOXSTRICTRC rcStrict = PGM_LOCK(pVM); /* We should already have it, but just make sure we do. */
471	if (RT_SUCCESS(rcStrict))
472	{
473	rcStrict = pgmPhysMmio2WriteHandlerCommon(pVM, pVCpu, uUser, GCPhys, ~(RTGCPTR)0);
474	PGM_UNLOCK(pVM);
475	if (rcStrict == VINF_SUCCESS)
476	rcStrict = VINF_PGM_HANDLER_DO_DEFAULT;
477	}
478	RT_NOREF(pvPhys, pvBuf, cbBuf, enmAccessType, enmOrigin);
479	return rcStrict;
480	}
481
482
483
484	/*********************************************************************************************************************************
485	* RAM Ranges *
486	*********************************************************************************************************************************/
487
488	#ifdef VBOX_STRICT
489	/**
490	* Asserts that the RAM range structures are sane.
491	*/
492	DECLHIDDEN(bool) pgmPhysAssertRamRangesLocked(PVMCC pVM, bool fInUpdate, bool fRamRelaxed)
493	{
494	bool fRet = true;
495
496	/*
497	* Check the generation ID. This is stable since we own the PGM lock.
498	*/
499	AssertStmt((pVM->pgm.s.RamRangeUnion.idGeneration & 1U) == (unsigned)fInUpdate, fRet = false);
500
501	/*
502	* Check the entry count and max ID.
503	*/
504	uint32_t const idRamRangeMax = pVM->pgm.s.idRamRangeMax;
505	/* Since this is set to the highest ID, it cannot be the same as the table size. */
506	AssertStmt(idRamRangeMax < RT_ELEMENTS(pVM->pgm.s.apRamRanges), fRet = false);
507
508	/* Because ID=0 is reserved, it's one less than the table size and at most the
509	same as the max ID. */
510	uint32_t const cLookupEntries = pVM->pgm.s.RamRangeUnion.cLookupEntries;
511	AssertStmt(cLookupEntries < RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup), fRet = false);
512	AssertStmt(cLookupEntries <= idRamRangeMax, fRet = false);
513
514	/*
515	* Check the pointer table(s).
516	*/
517	/* The first entry shall be empty. */
518	AssertStmt(pVM->pgm.s.apRamRanges[0] == NULL, fRet = false);
519	# ifdef IN_RING0
520	AssertStmt(pVM->pgmr0.s.apRamRanges[0] == NULL, fRet = false);
521	AssertStmt(pVM->pgmr0.s.acRamRangePages[0] == 0, fRet = false);
522	# endif
523
524	uint32_t cMappedRanges = 0;
525	for (uint32_t idRamRange = 1; idRamRange <= idRamRangeMax; idRamRange++)
526	{
527	# ifdef IN_RING0
528	PPGMRAMRANGE const pRamRange = pVM->pgmr0.s.apRamRanges[idRamRange];
529	AssertContinueStmt(pRamRange, fRet = false);
530	AssertStmt(pVM->pgm.s.apRamRanges[idRamRange] != NIL_RTR3PTR, fRet = false);
531	AssertStmt( (pRamRange->cb >> GUEST_PAGE_SHIFT) == pVM->pgmr0.s.acRamRangePages[idRamRange]
532	\|\| ( (pRamRange->cb >> GUEST_PAGE_SHIFT) < pVM->pgmr0.s.acRamRangePages[idRamRange]
533	&& !(pRamRange->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX)),
534	fRet = false);
535	# else
536	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apRamRanges[idRamRange];
537	AssertContinueStmt(pRamRange, fRet = false);
538	# endif
539	AssertStmt(pRamRange->idRange == idRamRange, fRet = false);
540	if (pRamRange->GCPhys != NIL_RTGCPHYS)
541	{
542	cMappedRanges++;
543	AssertStmt((pRamRange->GCPhys & GUEST_PAGE_OFFSET_MASK) == 0, fRet = false);
544	AssertStmt((pRamRange->GCPhysLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK, fRet = false);
545	AssertStmt(pRamRange->GCPhysLast > pRamRange->GCPhys, fRet = false);
546	AssertStmt(pRamRange->GCPhysLast - pRamRange->GCPhys + 1U == pRamRange->cb, fRet = false);
547	}
548	else
549	{
550	AssertStmt(pRamRange->GCPhysLast == NIL_RTGCPHYS, fRet = false);
551	AssertStmt(PGM_RAM_RANGE_IS_AD_HOC(pRamRange) \|\| fRamRelaxed, fRet = false);
552	}
553	}
554
555	/*
556	* Check that the lookup table is sorted and contains the right information.
557	*/
558	AssertMsgStmt(cMappedRanges == cLookupEntries,
559	("cMappedRanges=%#x cLookupEntries=%#x\n", cMappedRanges, cLookupEntries),
560	fRet = false);
561	RTGCPHYS GCPhysPrev = ~(RTGCPHYS)0;
562	for (uint32_t idxLookup = 0; idxLookup < cLookupEntries; idxLookup++)
563	{
564	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
565	AssertContinueStmt(idRamRange > 0 && idRamRange <= idRamRangeMax, fRet = false);
566	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm,pgmr0,pgmrc).s.apRamRanges[idRamRange];
567	AssertContinueStmt(pRamRange, fRet = false);
568
569	AssertStmt(pRamRange->idRange == idRamRange, fRet = false);
570	AssertStmt(pRamRange->GCPhys == PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]),
571	fRet = false);
572	AssertStmt(pRamRange->GCPhysLast == pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast, fRet = false);
573
574	AssertStmt(pRamRange->GCPhys >= GCPhysPrev + 1U, fRet = false);
575	GCPhysPrev = pRamRange->GCPhysLast;
576	}
577
578	return fRet;
579	}
580	#endif /* VBOX_STRICT */
581
582
583	/**
584	* Invalidates the RAM range TLBs.
585	*
586	* @param pVM The cross context VM structure.
587	*/
588	void pgmPhysInvalidRamRangeTlbs(PVMCC pVM)
589	{
590	PGM_LOCK_VOID(pVM);
591
592	/* This is technically only required when freeing the PCNet MMIO2 range
593	during ancient saved state loading. The code freeing the RAM range
594	will make sure this function is called in both rings. */
595	RT_ZERO(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb);
596	VMCC_FOR_EACH_VMCPU_STMT(pVM, RT_ZERO(pVCpu->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb));
597
598	PGM_UNLOCK(pVM);
599	}
600
601
602	/**
603	* Tests if a value of type RTGCPHYS is negative if the type had been signed
604	* instead of unsigned.
605	*
606	* @returns @c true if negative, @c false if positive or zero.
607	* @param a_GCPhys The value to test.
608	* @todo Move me to iprt/types.h.
609	*/
610	#define RTGCPHYS_IS_NEGATIVE(a_GCPhys) ((a_GCPhys) & ((RTGCPHYS)1 << (sizeof(RTGCPHYS)*8 - 1)))
611
612
613	/**
614	* Slow worker for pgmPhysGetRange.
615	*
616	* @copydoc pgmPhysGetRange
617	* @note Caller owns the PGM lock.
618	*/
619	DECLHIDDEN(PPGMRAMRANGE) pgmPhysGetRangeSlow(PVMCC pVM, RTGCPHYS GCPhys)
620	{
621	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
622
623	uint32_t idxEnd = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
624	uint32_t idxStart = 0;
625	for (;;)
626	{
627	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
628	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
629	RTGCPHYS const cbEntryMinus1 = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast - GCPhysEntryFirst;
630	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
631	if (off <= cbEntryMinus1)
632	{
633	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
634	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), NULL);
635	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
636	Assert(pRamRange);
637	pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
638	return pRamRange;
639	}
640	if (RTGCPHYS_IS_NEGATIVE(off))
641	{
642	if (idxStart < idxLookup)
643	idxEnd = idxLookup;
644	else
645	break;
646	}
647	else
648	{
649	idxLookup += 1;
650	if (idxLookup < idxEnd)
651	idxStart = idxLookup;
652	else
653	break;
654	}
655	}
656	return NULL;
657	}
658
659
660	/**
661	* Slow worker for pgmPhysGetRangeAtOrAbove.
662	*
663	* @copydoc pgmPhysGetRangeAtOrAbove
664	*/
665	DECLHIDDEN(PPGMRAMRANGE) pgmPhysGetRangeAtOrAboveSlow(PVMCC pVM, RTGCPHYS GCPhys)
666	{
667	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
668
669	uint32_t idRamRangeLastLeft = UINT32_MAX;
670	uint32_t idxEnd = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
671	uint32_t idxStart = 0;
672	for (;;)
673	{
674	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
675	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
676	RTGCPHYS const cbEntryMinus1 = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast - GCPhysEntryFirst;
677	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
678	if (off <= cbEntryMinus1)
679	{
680	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
681	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), NULL);
682	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
683	Assert(pRamRange);
684	pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
685	return pRamRange;
686	}
687	if (RTGCPHYS_IS_NEGATIVE(off))
688	{
689	idRamRangeLastLeft = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
690	if (idxStart < idxLookup)
691	idxEnd = idxLookup;
692	else
693	break;
694	}
695	else
696	{
697	idxLookup += 1;
698	if (idxLookup < idxEnd)
699	idxStart = idxLookup;
700	else
701	break;
702	}
703	}
704	if (idRamRangeLastLeft != UINT32_MAX)
705	{
706	AssertReturn(idRamRangeLastLeft < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), NULL);
707	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRangeLastLeft];
708	Assert(pRamRange);
709	return pRamRange;
710	}
711	return NULL;
712	}
713
714
715	/**
716	* Slow worker for pgmPhysGetPage.
717	*
718	* @copydoc pgmPhysGetPage
719	*/
720	DECLHIDDEN(PPGMPAGE) pgmPhysGetPageSlow(PVMCC pVM, RTGCPHYS GCPhys)
721	{
722	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
723
724	uint32_t idxEnd = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
725	uint32_t idxStart = 0;
726	for (;;)
727	{
728	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
729	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
730	RTGCPHYS const cbEntryMinus1 = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast - GCPhysEntryFirst;
731	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
732	if (off <= cbEntryMinus1)
733	{
734	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
735	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), NULL);
736	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
737	AssertReturn(pRamRange, NULL);
738	pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
739
740	/* Get the page. */
741	Assert(off < pRamRange->cb);
742	RTGCPHYS const idxPage = off >> GUEST_PAGE_SHIFT;
743	#ifdef IN_RING0
744	AssertReturn(idxPage < pVM->pgmr0.s.acRamRangePages[idRamRange], NULL);
745	#endif
746	return &pRamRange->aPages[idxPage];
747	}
748	if (RTGCPHYS_IS_NEGATIVE(off))
749	{
750	if (idxStart < idxLookup)
751	idxEnd = idxLookup;
752	else
753	break;
754	}
755	else
756	{
757	idxLookup += 1;
758	if (idxLookup < idxEnd)
759	idxStart = idxLookup;
760	else
761	break;
762	}
763	}
764	return NULL;
765	}
766
767
768	/**
769	* Slow worker for pgmPhysGetPageEx.
770	*
771	* @copydoc pgmPhysGetPageEx
772	*/
773	DECLHIDDEN(int) pgmPhysGetPageExSlow(PVMCC pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
774	{
775	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
776
777	uint32_t idxEnd = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
778	uint32_t idxStart = 0;
779	for (;;)
780	{
781	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
782	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
783	RTGCPHYS const cbEntryMinus1 = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast - GCPhysEntryFirst;
784	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
785	if (off <= cbEntryMinus1)
786	{
787	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
788	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), VERR_PGM_PHYS_RAM_LOOKUP_IPE);
789	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
790	AssertReturn(pRamRange, VERR_PGM_PHYS_RAM_LOOKUP_IPE);
791	pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
792
793	/* Get the page. */
794	Assert(off < pRamRange->cb);
795	RTGCPHYS const idxPage = off >> GUEST_PAGE_SHIFT;
796	#ifdef IN_RING0
797	AssertReturn(idxPage < pVM->pgmr0.s.acRamRangePages[idRamRange], VERR_PGM_PHYS_RAM_LOOKUP_IPE);
798	#endif
799	*ppPage = &pRamRange->aPages[idxPage];
800	return VINF_SUCCESS;
801	}
802	if (RTGCPHYS_IS_NEGATIVE(off))
803	{
804	if (idxStart < idxLookup)
805	idxEnd = idxLookup;
806	else
807	break;
808	}
809	else
810	{
811	idxLookup += 1;
812	if (idxLookup < idxEnd)
813	idxStart = idxLookup;
814	else
815	break;
816	}
817	}
818
819	*ppPage = NULL;
820	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
821	}
822
823
824	/**
825	* Slow worker for pgmPhysGetPageAndRangeEx.
826	*
827	* @copydoc pgmPhysGetPageAndRangeEx
828	*/
829	DECLHIDDEN(int) pgmPhysGetPageAndRangeExSlow(PVMCC pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
830	{
831	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
832
833	uint32_t idxEnd = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
834	uint32_t idxStart = 0;
835	for (;;)
836	{
837	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
838	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
839	RTGCPHYS const cbEntryMinus1 = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast - GCPhysEntryFirst;
840	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
841	if (off <= cbEntryMinus1)
842	{
843	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
844	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), VERR_PGM_PHYS_RAM_LOOKUP_IPE);
845	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
846	AssertReturn(pRamRange, VERR_PGM_PHYS_RAM_LOOKUP_IPE);
847	pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
848
849	/* Get the page. */
850	Assert(off < pRamRange->cb);
851	RTGCPHYS const idxPage = off >> GUEST_PAGE_SHIFT;
852	#ifdef IN_RING0
853	AssertReturn(idxPage < pVM->pgmr0.s.acRamRangePages[idRamRange], VERR_PGM_PHYS_RAM_LOOKUP_IPE);
854	#endif
855	*ppRam = pRamRange;
856	*ppPage = &pRamRange->aPages[idxPage];
857	return VINF_SUCCESS;
858	}
859	if (RTGCPHYS_IS_NEGATIVE(off))
860	{
861	if (idxStart < idxLookup)
862	idxEnd = idxLookup;
863	else
864	break;
865	}
866	else
867	{
868	idxLookup += 1;
869	if (idxLookup < idxEnd)
870	idxStart = idxLookup;
871	else
872	break;
873	}
874	}
875
876	*ppRam = NULL;
877	*ppPage = NULL;
878	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
879	}
880
881
882	/**
883	* Slow worker for pgmPhysGetPageAndRangeExLockless.
884	*
885	* @copydoc pgmPhysGetPageAndRangeExLockless
886	*/
887	DECLHIDDEN(int) pgmPhysGetPageAndRangeExSlowLockless(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys,
888	PGMPAGE volatile ppPage, PGMRAMRANGE volatile ppRam)
889	{
890	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,RamRangeTlbMisses));
891
892	PGM::PGMRAMRANGEGENANDLOOKUPCOUNT RamRangeUnion;
893	RamRangeUnion.u64Combined = ASMAtomicUoReadU64(&pVM->pgm.s.RamRangeUnion.u64Combined);
894
895	uint32_t idxEnd = RT_MIN(RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
896	uint32_t idxStart = 0;
897	for (;;)
898	{
899	/* Read the entry as atomically as possible: */
900	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
901	PGMRAMRANGELOOKUPENTRY Entry;
902	#if (RTASM_HAVE_READ_U128+0) & 1
903	Entry.u128Normal = ASMAtomicUoReadU128U(&pVM->pgm.s.aRamRangeLookup[idxLookup].u128Volatile);
904	#else
905	Entry.u128Normal.s.Lo = pVM->pgm.s.aRamRangeLookup[idxLookup].u128Volatile.s.Lo;
906	Entry.u128Normal.s.Hi = pVM->pgm.s.aRamRangeLookup[idxLookup].u128Volatile.s.Hi;
907	ASMCompilerBarrier(); /paranoia^2/
908	if (RT_LIKELY(Entry.u128Normal.s.Lo == pVM->pgm.s.aRamRangeLookup[idxLookup].u128Volatile.s.Lo))
909	{ /* likely */ }
910	else
911	break;
912	#endif
913
914	/* Check how GCPhys relates to the entry: */
915	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(Entry);
916	RTGCPHYS const cbEntryMinus1 = Entry.GCPhysLast - GCPhysEntryFirst;
917	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
918	if (off <= cbEntryMinus1)
919	{
920	/* We seem to have a match. If, however, anything doesn't match up
921	bail and redo owning the lock. No asserting here as we may be
922	racing removal/insertion. */
923	if (!RTGCPHYS_IS_NEGATIVE(off))
924	{
925	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(Entry);
926	if (idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges))
927	{
928	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
929	if (pRamRange)
930	{
931	if ( pRamRange->GCPhys == GCPhysEntryFirst
932	&& pRamRange->cb == cbEntryMinus1 + 1U)
933	{
934	RTGCPHYS const idxPage = off >> GUEST_PAGE_SHIFT;
935	#ifdef IN_RING0
936	if (idxPage < pVM->pgmr0.s.acRamRangePages[idRamRange])
937	#endif
938	{
939	pVCpu->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
940	*ppRam = pRamRange;
941	*ppPage = &pRamRange->aPages[idxPage];
942	return VINF_SUCCESS;
943	}
944	}
945	}
946	}
947	}
948	break;
949	}
950	if (RTGCPHYS_IS_NEGATIVE(off))
951	{
952	if (idxStart < idxLookup)
953	idxEnd = idxLookup;
954	else
955	break;
956	}
957	else
958	{
959	idxLookup += 1;
960	if (idxLookup < idxEnd)
961	idxStart = idxLookup;
962	else
963	break;
964	}
965	}
966
967	/*
968	* If we get down here, we do the lookup again but while owning the PGM lock.
969	*/
970	*ppRam = NULL;
971	*ppPage = NULL;
972	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,RamRangeTlbLocking));
973
974	PGM_LOCK_VOID(pVM);
975	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, (PPGMPAGE )ppPage, (PPGMRAMRANGE )ppRam);
976	PGM_UNLOCK(pVM);
977
978	PGMRAMRANGE volatile * const pRam = *ppRam;
979	if (pRam)
980	pVCpu->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = (PPGMRAMRANGE)pRam;
981	return rc;
982	}
983
984
985	/**
986	* Common worker for pgmR3PhysAllocateRamRange, PGMR0PhysAllocateRamRangeReq,
987	* and pgmPhysMmio2RegisterWorker2.
988	*/
989	DECLHIDDEN(int) pgmPhysRamRangeAllocCommon(PVMCC pVM, uint32_t cPages, uint32_t fFlags, uint32_t *pidNewRange)
990	{
991
992	/*
993	* Allocate the RAM range structure and map it into ring-3.
994	*/
995	size_t const cbRamRange = RT_ALIGN_Z(RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]), HOST_PAGE_SIZE);
996	#ifdef IN_RING0
997	RTR0MEMOBJ hMemObj = NIL_RTR0MEMOBJ;
998	int rc = RTR0MemObjAllocPage(&hMemObj, cbRamRange, false /fExecutable/);
999	#else
1000	PPGMRAMRANGE pRamRange;
1001	int rc = SUPR3PageAlloc(cbRamRange >> HOST_PAGE_SHIFT, 0 /fFlags/, (void **)&pRamRange);
1002	#endif
1003	if (RT_SUCCESS(rc))
1004	{
1005	/* Zero the memory and do basic range init before mapping it into userland. */
1006	#ifdef IN_RING0
1007	PPGMRAMRANGE const pRamRange = (PPGMRAMRANGE)RTR0MemObjAddress(hMemObj);
1008	if (!RTR0MemObjWasZeroInitialized(hMemObj))
1009	#endif
1010	RT_BZERO(pRamRange, cbRamRange);
1011
1012	pRamRange->GCPhys = NIL_RTGCPHYS;
1013	pRamRange->cb = (RTGCPHYS)cPages << GUEST_PAGE_SHIFT;
1014	pRamRange->GCPhysLast = NIL_RTGCPHYS;
1015	pRamRange->fFlags = fFlags;
1016	pRamRange->idRange = UINT32_MAX / 2;
1017
1018	#ifdef IN_RING0
1019	/* Map it into userland. */
1020	RTR0MEMOBJ hMapObj = NIL_RTR0MEMOBJ;
1021	rc = RTR0MemObjMapUser(&hMapObj, hMemObj, (RTR3PTR)-1, 0 /uAlignment/,
1022	RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
1023	if (RT_SUCCESS(rc))
1024	#endif
1025	{
1026	/*
1027	* Grab the lock (unlikely to fail or block as caller typically owns it already).
1028	*/
1029	rc = PGM_LOCK(pVM);
1030	if (RT_SUCCESS(rc))
1031	{
1032	/*
1033	* Allocate a range ID.
1034	*/
1035	uint32_t idRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.idRamRangeMax + 1;
1036	if (idRamRange != 0 && idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges))
1037	{
1038	#ifdef IN_RING0
1039	if (pVM->pgmr0.s.apRamRanges[idRamRange] == NULL)
1040	#endif
1041	{
1042	if (pVM->pgm.s.apRamRanges[idRamRange] == NIL_RTR3PTR)
1043	{
1044	/*
1045	* Commit it.
1046	*/
1047	#ifdef IN_RING0
1048	pVM->pgmr0.s.apRamRanges[idRamRange] = pRamRange;
1049	pVM->pgmr0.s.acRamRangePages[idRamRange] = cPages;
1050	pVM->pgmr0.s.ahRamRangeMemObjs[idRamRange] = hMemObj;
1051	pVM->pgmr0.s.ahRamRangeMapObjs[idRamRange] = hMapObj;
1052	pVM->pgmr0.s.idRamRangeMax = idRamRange;
1053	#endif
1054
1055	pVM->pgm.s.idRamRangeMax = idRamRange;
1056	#ifdef IN_RING0
1057	pVM->pgm.s.apRamRanges[idRamRange] = RTR0MemObjAddressR3(hMapObj);
1058	#else
1059	pVM->pgm.s.apRamRanges[idRamRange] = pRamRange;
1060	#endif
1061
1062	pRamRange->idRange = idRamRange;
1063	*pidNewRange = idRamRange;
1064
1065	PGM_UNLOCK(pVM);
1066	return VINF_SUCCESS;
1067	}
1068	}
1069
1070	/*
1071	* Bail out.
1072	*/
1073	rc = VERR_INTERNAL_ERROR_5;
1074	}
1075	else
1076	rc = VERR_PGM_TOO_MANY_RAM_RANGES;
1077	PGM_UNLOCK(pVM);
1078	}
1079	#ifdef IN_RING0
1080	RTR0MemObjFree(hMapObj, false /fFreeMappings/);
1081	#endif
1082	}
1083	#ifdef IN_RING0
1084	RTR0MemObjFree(hMemObj, true /fFreeMappings/);
1085	#else
1086	SUPR3PageFree(pRamRange, cbRamRange >> HOST_PAGE_SHIFT);
1087	#endif
1088	}
1089	*pidNewRange = UINT32_MAX;
1090	return rc;
1091	}
1092
1093
1094	#ifdef IN_RING0
1095	/**
1096	* This is called during VM initialization to allocate a RAM range.
1097	*
1098	* The range is not entered into the lookup table, that is something the caller
1099	* has to do. The PGMPAGE entries are zero'ed, but otherwise uninitialized.
1100	*
1101	* @returns VBox status code.
1102	* @param pGVM Pointer to the global VM structure.
1103	* @param pReq Where to get the parameters and return the range ID.
1104	* @thread EMT(0)
1105	*/
1106	VMMR0_INT_DECL(int) PGMR0PhysAllocateRamRangeReq(PGVM pGVM, PPGMPHYSALLOCATERAMRANGEREQ pReq)
1107	{
1108	/*
1109	* Validate input (ASSUME pReq is a copy and can't be modified by ring-3
1110	* while we're here).
1111	*/
1112	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
1113	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x < %#zx\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
1114
1115	AssertReturn(pReq->cbGuestPage == GUEST_PAGE_SIZE, VERR_INCOMPATIBLE_CONFIG);
1116
1117	AssertReturn(pReq->cGuestPages > 0, VERR_OUT_OF_RANGE);
1118	AssertReturn(pReq->cGuestPages <= PGM_MAX_PAGES_PER_RAM_RANGE, VERR_OUT_OF_RANGE);
1119
1120	AssertMsgReturn(!(pReq->fFlags & ~(uint32_t)PGM_RAM_RANGE_FLAGS_VALID_MASK), ("fFlags=%#RX32\n", pReq->fFlags),
1121	VERR_INVALID_FLAGS);
1122
1123	/** @todo better VM state guard, enmVMState is ring-3 writable. */
1124	VMSTATE const enmState = pGVM->enmVMState;
1125	AssertMsgReturn(enmState == VMSTATE_CREATING, ("enmState=%d\n", enmState), VERR_VM_INVALID_VM_STATE);
1126	VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
1127
1128	/*
1129	* Call common worker.
1130	*/
1131	return pgmPhysRamRangeAllocCommon(pGVM, pReq->cGuestPages, pReq->fFlags, &pReq->idNewRange);
1132	}
1133	#endif /* IN_RING0 */
1134
1135
1136	/**
1137	* Frees a RAM range.
1138	*
1139	* This is not a typical occurence. Currently only used for a special MMIO2
1140	* saved state compatibility scenario involving PCNet and state saved before
1141	* VBox v4.3.6.
1142	*/
1143	static int pgmPhysRamRangeFree(PVMCC pVM, PPGMRAMRANGE pRamRange)
1144	{
1145	/*
1146	* Some basic input validation.
1147	*/
1148	AssertPtrReturn(pRamRange, VERR_INVALID_PARAMETER);
1149	uint32_t const idRamRange = ASMAtomicReadU32(&pRamRange->idRange);
1150	ASMCompilerBarrier();
1151	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), VERR_INVALID_PARAMETER);
1152	AssertReturn(pRamRange == pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange], VERR_INVALID_PARAMETER);
1153	AssertReturn(pRamRange->GCPhys == NIL_RTGCPHYS, VERR_RESOURCE_BUSY);
1154
1155	/*
1156	* Kill the range pointers and associated data.
1157	*/
1158	pVM->pgm.s.apRamRanges[idRamRange] = NIL_RTR3PTR;
1159	#ifdef IN_RING0
1160	pVM->pgmr0.s.apRamRanges[idRamRange] = NULL;
1161	#endif
1162
1163	/*
1164	* Zap the pages and other RAM ranges properties to ensure there aren't any
1165	* stale references to anything hanging around should the freeing go awry.
1166	*/
1167	#ifdef IN_RING0
1168	uint32_t const cPages = pVM->pgmr0.s.acRamRangePages[idRamRange];
1169	pVM->pgmr0.s.acRamRangePages[idRamRange] = 0;
1170	#else
1171	uint32_t const cPages = pRamRange->cb >> GUEST_PAGE_SHIFT;
1172	#endif
1173	RT_BZERO(pRamRange->aPages, cPages * sizeof(pRamRange->aPages[0]));
1174
1175	pRamRange->fFlags = UINT32_MAX;
1176	pRamRange->cb = NIL_RTGCPHYS;
1177	pRamRange->pbR3 = NIL_RTR3PTR;
1178	pRamRange->pszDesc = NIL_RTR3PTR;
1179	pRamRange->paLSPages = NIL_RTR3PTR;
1180	pRamRange->idRange = UINT32_MAX / 8;
1181
1182	/*
1183	* Free the RAM range itself.
1184	*/
1185	#ifdef IN_RING0
1186	Assert(pVM->pgmr0.s.ahRamRangeMapObjs[idRamRange] != NIL_RTR0MEMOBJ);
1187	int rc = RTR0MemObjFree(pVM->pgmr0.s.ahRamRangeMapObjs[idRamRange], true /fFreeMappings/);
1188	if (RT_SUCCESS(rc))
1189	{
1190	pVM->pgmr0.s.ahRamRangeMapObjs[idRamRange] = NIL_RTR0MEMOBJ;
1191	rc = RTR0MemObjFree(pVM->pgmr0.s.ahRamRangeMemObjs[idRamRange], true /fFreeMappings/);
1192	if (RT_SUCCESS(rc))
1193	pVM->pgmr0.s.ahRamRangeMemObjs[idRamRange] = NIL_RTR0MEMOBJ;
1194	}
1195	#else
1196	size_t const cbRamRange = RT_ALIGN_Z(RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]), HOST_PAGE_SIZE);
1197	int rc = SUPR3PageFree(pRamRange, cbRamRange >> HOST_PAGE_SHIFT);
1198	#endif
1199
1200	/*
1201	* Decrease the max ID if removal was successful and this was the final
1202	* RAM range entry.
1203	*/
1204	if ( RT_SUCCESS(rc)
1205	&& idRamRange == pVM->CTX_EXPR(pgm, pgmr0, pgm).s.idRamRangeMax)
1206	{
1207	pVM->pgm.s.idRamRangeMax = idRamRange - 1;
1208	#ifdef IN_RING0
1209	pVM->pgmr0.s.idRamRangeMax = idRamRange - 1;
1210	#endif
1211	}
1212
1213	/*
1214	* Make sure the RAM range TLB does not contain any stale pointers to this range.
1215	*/
1216	pgmPhysInvalidRamRangeTlbs(pVM);
1217	return rc;
1218	}
1219
1220
1221
1222	/*********************************************************************************************************************************
1223	* MMIO2 *
1224	*********************************************************************************************************************************/
1225
1226	/**
1227	* Calculates the number of chunks
1228	*
1229	* @returns Number of registration chunk needed.
1230	* @param cb The size of the MMIO/MMIO2 range.
1231	* @param pcPagesPerChunk Where to return the number of guest pages tracked by
1232	* each chunk. Optional.
1233	*/
1234	DECLHIDDEN(uint16_t) pgmPhysMmio2CalcChunkCount(RTGCPHYS cb, uint32_t *pcPagesPerChunk)
1235	{
1236	/*
1237	* This is the same calculation as PGMR3PhysRegisterRam does, except we'll be
1238	* needing a few bytes extra the PGMREGMMIO2RANGE structure.
1239	*
1240	* Note! In additions, we've got a 24 bit sub-page range for MMIO2 ranges, leaving
1241	* us with an absolute maximum of 16777215 pages per chunk (close to 64 GB).
1242	*/
1243	AssertCompile(PGM_MAX_PAGES_PER_RAM_RANGE < _16M);
1244	uint32_t const cPagesPerChunk = PGM_MAX_PAGES_PER_RAM_RANGE;
1245
1246	if (pcPagesPerChunk)
1247	*pcPagesPerChunk = cPagesPerChunk;
1248
1249	/* Calc the number of chunks we need. */
1250	RTGCPHYS const cGuestPages = cb >> GUEST_PAGE_SHIFT;
1251	uint16_t cChunks = (uint16_t)((cGuestPages + cPagesPerChunk - 1) / cPagesPerChunk);
1252	#ifdef IN_RING3
1253	AssertRelease((RTGCPHYS)cChunks * cPagesPerChunk >= cGuestPages);
1254	#else
1255	AssertReturn((RTGCPHYS)cChunks * cPagesPerChunk >= cGuestPages, 0);
1256	#endif
1257	return cChunks;
1258	}
1259
1260
1261	/**
1262	* Worker for PGMR3PhysMmio2Register and PGMR0PhysMmio2RegisterReq.
1263	*
1264	* (The caller already know which MMIO2 region ID will be assigned and how many
1265	* chunks will be used, so no output parameters required.)
1266	*/
1267	DECLHIDDEN(int) pgmPhysMmio2RegisterWorker(PVMCC pVM, uint32_t const cGuestPages, uint8_t const idMmio2,
1268	const uint8_t cChunks, PPDMDEVINSR3 const pDevIns, uint8_t
1269	const iSubDev, uint8_t const iRegion, uint32_t const fFlags)
1270	{
1271	/*
1272	* Get the number of pages per chunk.
1273	*/
1274	uint32_t cGuestPagesPerChunk;
1275	AssertReturn(pgmPhysMmio2CalcChunkCount((RTGCPHYS)cGuestPages << GUEST_PAGE_SHIFT, &cGuestPagesPerChunk) == cChunks,
1276	VERR_PGM_PHYS_MMIO_EX_IPE);
1277	Assert(idMmio2 != 0);
1278
1279	/*
1280	* The first thing we need to do is the allocate the memory that will be
1281	* backing the whole range.
1282	*/
1283	RTGCPHYS const cbMmio2Backing = (RTGCPHYS)cGuestPages << GUEST_PAGE_SHIFT;
1284	uint32_t const cHostPages = (cbMmio2Backing + HOST_PAGE_SIZE - 1U) >> HOST_PAGE_SHIFT;
1285	size_t const cbMmio2Aligned = cHostPages << HOST_PAGE_SHIFT;
1286	R3PTRTYPE(uint8_t *) pbMmio2BackingR3 = NIL_RTR3PTR;
1287	#ifdef IN_RING0
1288	RTR0MEMOBJ hMemObj = NIL_RTR0MEMOBJ;
1289	# ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
1290	int rc = RTR0MemObjAllocPage(&hMemObj, cbMmio2Aligned, false /fExecutable/);
1291	# else
1292	int rc = RTR0MemObjAllocPhysNC(&hMemObj, cbMmio2Aligned, NIL_RTHCPHYS);
1293	# endif
1294	#else /* !IN_RING0 */
1295	AssertReturn(PGM_IS_IN_NEM_MODE(pVM), VERR_INTERNAL_ERROR_4);
1296	int rc = SUPR3PageAlloc(cHostPages, pVM->pgm.s.fUseLargePages ? SUP_PAGE_ALLOC_F_LARGE_PAGES : 0, (void **)&pbMmio2BackingR3);
1297	#endif /* !IN_RING0 */
1298	if (RT_SUCCESS(rc))
1299	{
1300	/*
1301	* Make sure it's is initialized to zeros before it's mapped to userland.
1302	*/
1303	#ifdef IN_RING0
1304	# ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
1305	uint8_t pbMmio2BackingR0 = (uint8_t )RTR0MemObjAddress(hMemObj);
1306	AssertPtr(pbMmio2BackingR0);
1307	# endif
1308	rc = RTR0MemObjZeroInitialize(hMemObj, false /fForce/);
1309	AssertRCReturnStmt(rc, RTR0MemObjFree(hMemObj, true /fFreeMappings/), rc);
1310	#else
1311	RT_BZERO(pbMmio2BackingR3, cbMmio2Aligned);
1312	#endif
1313
1314	#ifdef IN_RING0
1315	/*
1316	* Map it into ring-3.
1317	*/
1318	RTR0MEMOBJ hMapObj = NIL_RTR0MEMOBJ;
1319	rc = RTR0MemObjMapUser(&hMapObj, hMemObj, (RTR3PTR)-1, 0, RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
1320	if (RT_SUCCESS(rc))
1321	{
1322	pbMmio2BackingR3 = RTR0MemObjAddressR3(hMapObj);
1323	#endif
1324
1325	/*
1326	* Create the MMIO2 registration records and associated RAM ranges.
1327	* The RAM range allocation may fail here.
1328	*/
1329	RTGCPHYS offMmio2Backing = 0;
1330	uint32_t cGuestPagesLeft = cGuestPages;
1331	for (uint32_t iChunk = 0, idx = idMmio2 - 1; iChunk < cChunks; iChunk++, idx++)
1332	{
1333	uint32_t const cPagesTrackedByChunk = RT_MIN(cGuestPagesLeft, cGuestPagesPerChunk);
1334
1335	/*
1336	* Allocate the RAM range for this chunk.
1337	*/
1338	uint32_t idRamRange = UINT32_MAX;
1339	rc = pgmPhysRamRangeAllocCommon(pVM, cPagesTrackedByChunk, PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX, &idRamRange);
1340	if (RT_FAILURE(rc))
1341	{
1342	/* We only zap the pointers to the backing storage.
1343	PGMR3Term and friends will clean up the RAM ranges and stuff. */
1344	while (iChunk-- > 0)
1345	{
1346	idx--;
1347	#ifdef IN_RING0
1348	pVM->pgmr0.s.acMmio2RangePages[idx] = 0;
1349	# ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
1350	pVM->pgmr0.s.apbMmio2Backing[idx] = NULL;
1351	# endif
1352	#endif
1353
1354	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
1355	pMmio2->pbR3 = NIL_RTR3PTR;
1356
1357	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apMmio2RamRanges[idx];
1358	pRamRange->pbR3 = NIL_RTR3PTR;
1359	RT_BZERO(&pRamRange->aPages[0], sizeof(pRamRange->aPages) * cGuestPagesPerChunk);
1360	}
1361	break;
1362	}
1363
1364	pVM->pgm.s.apMmio2RamRanges[idx] = pVM->pgm.s.apRamRanges[idRamRange];
1365	#ifdef IN_RING0
1366	pVM->pgmr0.s.apMmio2RamRanges[idx] = pVM->pgmr0.s.apRamRanges[idRamRange];
1367	pVM->pgmr0.s.acMmio2RangePages[idx] = cPagesTrackedByChunk;
1368	#endif
1369
1370	/* Initialize the RAM range. */
1371	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
1372	pRamRange->pbR3 = pbMmio2BackingR3 + offMmio2Backing;
1373	uint32_t iDstPage = cPagesTrackedByChunk;
1374	#ifdef IN_RING0
1375	AssertRelease(HOST_PAGE_SHIFT == GUEST_PAGE_SHIFT);
1376	while (iDstPage-- > 0)
1377	{
1378	RTHCPHYS HCPhys = RTR0MemObjGetPagePhysAddr(hMemObj, iDstPage + (offMmio2Backing >> HOST_PAGE_SHIFT));
1379	Assert(HCPhys != NIL_RTHCPHYS);
1380	PGM_PAGE_INIT(&pRamRange->aPages[iDstPage], HCPhys, PGM_MMIO2_PAGEID_MAKE(idMmio2, iDstPage),
1381	PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
1382	}
1383	#else
1384	Assert(PGM_IS_IN_NEM_MODE(pVM));
1385	while (iDstPage-- > 0)
1386	PGM_PAGE_INIT(&pRamRange->aPages[iDstPage], UINT64_C(0x0000ffffffff0000),
1387	PGM_MMIO2_PAGEID_MAKE(idMmio2, iDstPage),
1388	PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
1389	#endif
1390
1391	/*
1392	* Initialize the MMIO2 registration structure.
1393	*/
1394	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
1395	pMmio2->pDevInsR3 = pDevIns;
1396	pMmio2->pbR3 = pbMmio2BackingR3 + offMmio2Backing;
1397	pMmio2->fFlags = 0;
1398	if (iChunk == 0)
1399	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_FIRST_CHUNK;
1400	if (iChunk + 1 == cChunks)
1401	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_LAST_CHUNK;
1402	if (fFlags & PGMPHYS_MMIO2_FLAGS_TRACK_DIRTY_PAGES)
1403	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES;
1404
1405	pMmio2->iSubDev = iSubDev;
1406	pMmio2->iRegion = iRegion;
1407	pMmio2->idSavedState = UINT8_MAX;
1408	pMmio2->idMmio2 = idMmio2 + iChunk;
1409	pMmio2->idRamRange = idRamRange;
1410	Assert(pMmio2->idRamRange == idRamRange);
1411	pMmio2->GCPhys = NIL_RTGCPHYS;
1412	pMmio2->cbReal = (RTGCPHYS)cPagesTrackedByChunk << GUEST_PAGE_SHIFT;
1413	pMmio2->pPhysHandlerR3 = NIL_RTR3PTR; /* Pre-alloc is done by ring-3 caller. */
1414	pMmio2->paLSPages = NIL_RTR3PTR;
1415
1416	#if defined(IN_RING0) && !defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
1417	pVM->pgmr0.s.apbMmio2Backing[idx] = &pbMmio2BackingR0[offMmio2Backing];
1418	#endif
1419
1420	/* Advance */
1421	cGuestPagesLeft -= cPagesTrackedByChunk;
1422	offMmio2Backing += (RTGCPHYS)cPagesTrackedByChunk << GUEST_PAGE_SHIFT;
1423	} /* chunk alloc loop */
1424	Assert(cGuestPagesLeft == 0 \|\| RT_FAILURE_NP(rc));
1425	if (RT_SUCCESS(rc))
1426	{
1427	/*
1428	* Account for pages and ring-0 memory objects.
1429	*/
1430	pVM->pgm.s.cAllPages += cGuestPages;
1431	pVM->pgm.s.cPrivatePages += cGuestPages;
1432	#ifdef IN_RING0
1433	pVM->pgmr0.s.ahMmio2MemObjs[idMmio2 - 1] = hMemObj;
1434	pVM->pgmr0.s.ahMmio2MapObjs[idMmio2 - 1] = hMapObj;
1435	#endif
1436	pVM->pgm.s.cMmio2Ranges = idMmio2 + cChunks - 1U;
1437
1438	/*
1439	* Done!.
1440	*/
1441	return VINF_SUCCESS;
1442	}
1443
1444	/*
1445	* Bail.
1446	*/
1447	#ifdef IN_RING0
1448	RTR0MemObjFree(hMapObj, true /fFreeMappings/);
1449	}
1450	RTR0MemObjFree(hMemObj, true /fFreeMappings/);
1451	#else
1452	SUPR3PageFree(pbMmio2BackingR3, cHostPages);
1453	#endif
1454	}
1455	else
1456	LogRel(("pgmPhysMmio2RegisterWorker: Failed to allocate %RGp bytes of MMIO2 backing memory: %Rrc\n", cbMmio2Aligned, rc));
1457	return rc;
1458	}
1459
1460
1461	#ifdef IN_RING0
1462	/**
1463	* This is called during VM initialization to create an MMIO2 range.
1464	*
1465	* This does everything except setting the PGMRAMRANGE::pszDesc to a non-zero
1466	* value and preallocating the access handler for dirty bitmap tracking.
1467	*
1468	* The caller already knows which MMIO2 ID will be assigned to the registration
1469	* and how many chunks it requires, so there are no output fields in the request
1470	* structure.
1471	*
1472	* @returns VBox status code.
1473	* @param pGVM Pointer to the global VM structure.
1474	* @param pReq Where to get the parameters.
1475	* @thread EMT(0)
1476	*/
1477	VMMR0_INT_DECL(int) PGMR0PhysMmio2RegisterReq(PGVM pGVM, PPGMPHYSMMIO2REGISTERREQ pReq)
1478	{
1479	/*
1480	* Validate input (ASSUME pReq is a copy and can't be modified by ring-3
1481	* while we're here).
1482	*/
1483	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
1484	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x < %#zx\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
1485
1486	/** @todo better VM state guard, enmVMState is ring-3 writable. */
1487	VMSTATE const enmState = pGVM->enmVMState;
1488	AssertMsgReturn( enmState == VMSTATE_CREATING
1489	\|\| enmState == VMSTATE_LOADING /* pre 4.3.6 state loading needs to ignore a MMIO2 region in PCNet. */
1490	, ("enmState=%d\n", enmState), VERR_VM_INVALID_VM_STATE);
1491	VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
1492
1493	AssertReturn(pReq->cbGuestPage == GUEST_PAGE_SIZE, VERR_INCOMPATIBLE_CONFIG);
1494	AssertReturn(GUEST_PAGE_SIZE == HOST_PAGE_SIZE, VERR_INCOMPATIBLE_CONFIG);
1495
1496	AssertReturn(pReq->cGuestPages > 0, VERR_OUT_OF_RANGE);
1497	AssertReturn(pReq->cGuestPages <= PGM_MAX_PAGES_PER_MMIO2_REGION, VERR_OUT_OF_RANGE);
1498	AssertReturn(pReq->cGuestPages <= (MM_MMIO_64_MAX >> GUEST_PAGE_SHIFT), VERR_OUT_OF_RANGE);
1499
1500	AssertMsgReturn(!(pReq->fFlags & ~PGMPHYS_MMIO2_FLAGS_VALID_MASK), ("fFlags=%#x\n", pReq->fFlags), VERR_INVALID_FLAGS);
1501
1502	AssertMsgReturn( pReq->cChunks > 0
1503	&& pReq->cChunks < PGM_MAX_MMIO2_RANGES
1504	&& pReq->cChunks == pgmPhysMmio2CalcChunkCount((RTGCPHYS)pReq->cGuestPages << GUEST_PAGE_SHIFT, NULL),
1505	("cChunks=%#x cGuestPages=%#x\n", pReq->cChunks, pReq->cGuestPages),
1506	VERR_INVALID_PARAMETER);
1507
1508	AssertMsgReturn( pReq->idMmio2 != 0
1509	&& pReq->idMmio2 <= PGM_MAX_MMIO2_RANGES
1510	&& (unsigned)pReq->idMmio2 + pReq->cChunks - 1U <= PGM_MAX_MMIO2_RANGES,
1511	("idMmio2=%#x cChunks=%#x\n", pReq->idMmio2, pReq->cChunks),
1512	VERR_INVALID_PARAMETER);
1513
1514	for (uint32_t iChunk = 0, idx = pReq->idMmio2 - 1; iChunk < pReq->cChunks; iChunk++, idx++)
1515	{
1516	AssertReturn(pGVM->pgmr0.s.ahMmio2MapObjs[idx] == NIL_RTR0MEMOBJ, VERR_INVALID_STATE);
1517	AssertReturn(pGVM->pgmr0.s.ahMmio2MemObjs[idx] == NIL_RTR0MEMOBJ, VERR_INVALID_STATE);
1518	AssertReturn(pGVM->pgmr0.s.apMmio2RamRanges[idx] == NULL, VERR_INVALID_STATE);
1519	}
1520
1521	/*
1522	* Make sure we're owning the PGM lock (caller should be), recheck idMmio2
1523	* and call the worker function we share with ring-3.
1524	*/
1525	int rc = PGM_LOCK(pGVM);
1526	AssertRCReturn(rc, rc);
1527
1528	AssertReturnStmt(pGVM->pgm.s.cMmio2Ranges + 1U == pReq->idMmio2,
1529	PGM_UNLOCK(pGVM), VERR_INVALID_PARAMETER);
1530	AssertReturnStmt(pGVM->pgmr0.s.idRamRangeMax + 1U + pReq->cChunks <= RT_ELEMENTS(pGVM->pgmr0.s.apRamRanges),
1531	PGM_UNLOCK(pGVM), VERR_PGM_TOO_MANY_RAM_RANGES);
1532
1533	rc = pgmPhysMmio2RegisterWorker(pGVM, pReq->cGuestPages, pReq->idMmio2, pReq->cChunks,
1534	pReq->pDevIns, pReq->iSubDev, pReq->iRegion, pReq->fFlags);
1535
1536	PGM_UNLOCK(pGVM);
1537	return rc;
1538	}
1539	#endif /* IN_RING0 */
1540
1541
1542
1543	/**
1544	* Worker for PGMR3PhysMmio2Deregister & PGMR0PhysMmio2DeregisterReq.
1545	*/
1546	DECLHIDDEN(int) pgmPhysMmio2DeregisterWorker(PVMCC pVM, uint8_t idMmio2, uint8_t cChunks, PPDMDEVINSR3 pDevIns)
1547	{
1548	/*
1549	* The caller shall have made sure all this is true, but we check again
1550	* since we're paranoid.
1551	*/
1552	AssertReturn(idMmio2 > 0 && idMmio2 <= RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges), VERR_INTERNAL_ERROR_2);
1553	AssertReturn(cChunks >= 1, VERR_INTERNAL_ERROR_2);
1554	uint8_t const idxFirst = idMmio2 - 1U;
1555	AssertReturn(idxFirst + cChunks <= pVM->pgm.s.cMmio2Ranges, VERR_INTERNAL_ERROR_2);
1556	uint32_t cGuestPages = 0; /* (For accounting and calulating backing memory size) */
1557	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
1558	{
1559	AssertReturn(pVM->pgm.s.aMmio2Ranges[idx].pDevInsR3 == pDevIns, VERR_NOT_OWNER);
1560	AssertReturn(!(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_MAPPED), VERR_RESOURCE_BUSY);
1561	AssertReturn(pVM->pgm.s.aMmio2Ranges[idx].GCPhys == NIL_RTGCPHYS, VERR_INVALID_STATE);
1562	if (iChunk == 0)
1563	AssertReturn(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK, VERR_INVALID_PARAMETER);
1564	else
1565	AssertReturn(!(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK), VERR_INVALID_PARAMETER);
1566	if (iChunk + 1 == cChunks)
1567	AssertReturn(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK, VERR_INVALID_PARAMETER);
1568	else
1569	AssertReturn(!(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK), VERR_INVALID_PARAMETER);
1570	AssertReturn(pVM->pgm.s.aMmio2Ranges[idx].pPhysHandlerR3 == NIL_RTR3PTR, VERR_INVALID_STATE); /* caller shall free this */
1571
1572	#ifdef IN_RING0
1573	cGuestPages += pVM->pgmr0.s.acMmio2RangePages[idx];
1574	#else
1575	cGuestPages += pVM->pgm.s.aMmio2Ranges[idx].cbReal >> GUEST_PAGE_SHIFT;
1576	#endif
1577
1578	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apMmio2RamRanges[idx];
1579	AssertPtrReturn(pRamRange, VERR_INVALID_STATE);
1580	AssertReturn(pRamRange->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX, VERR_INVALID_STATE);
1581	AssertReturn(pRamRange->GCPhys == NIL_RTGCPHYS, VERR_INVALID_STATE);
1582	AssertReturn(pRamRange->GCPhysLast == NIL_RTGCPHYS, VERR_INVALID_STATE);
1583	}
1584
1585	/*
1586	* Remove everything except the backing memory first. We work the ranges
1587	* in reverse so that we can reduce the max RAM range ID when possible.
1588	*/
1589	#ifdef IN_RING3
1590	uint8_t * const pbMmio2Backing = pVM->pgm.s.aMmio2Ranges[idxFirst].pbR3;
1591	RTGCPHYS const cbMmio2Backing = RT_ALIGN_T((RTGCPHYS)cGuestPages << GUEST_PAGE_SHIFT, HOST_PAGE_SIZE, RTGCPHYS);
1592	#endif
1593
1594	int rc = VINF_SUCCESS;
1595	uint32_t iChunk = cChunks;
1596	while (iChunk-- > 0)
1597	{
1598	uint32_t const idx = idxFirst + iChunk;
1599	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apMmio2RamRanges[idx];
1600
1601	/* Zap the MMIO2 region data. */
1602	pVM->pgm.s.apMmio2RamRanges[idx] = NIL_RTR3PTR;
1603	#ifdef IN_RING0
1604	pVM->pgmr0.s.apMmio2RamRanges[idx] = NULL;
1605	pVM->pgmr0.s.acMmio2RangePages[idx] = 0;
1606	#endif
1607	pVM->pgm.s.aMmio2Ranges[idx].pDevInsR3 = NIL_RTR3PTR;
1608	pVM->pgm.s.aMmio2Ranges[idx].pbR3 = NIL_RTR3PTR;
1609	pVM->pgm.s.aMmio2Ranges[idx].fFlags = 0;
1610	pVM->pgm.s.aMmio2Ranges[idx].iSubDev = UINT8_MAX;
1611	pVM->pgm.s.aMmio2Ranges[idx].iRegion = UINT8_MAX;
1612	pVM->pgm.s.aMmio2Ranges[idx].idSavedState = UINT8_MAX;
1613	pVM->pgm.s.aMmio2Ranges[idx].idMmio2 = UINT8_MAX;
1614	pVM->pgm.s.aMmio2Ranges[idx].idRamRange = UINT16_MAX;
1615	pVM->pgm.s.aMmio2Ranges[idx].GCPhys = NIL_RTGCPHYS;
1616	pVM->pgm.s.aMmio2Ranges[idx].cbReal = 0;
1617	pVM->pgm.s.aMmio2Ranges[idx].pPhysHandlerR3 = NIL_RTR3PTR;
1618	pVM->pgm.s.aMmio2Ranges[idx].paLSPages = NIL_RTR3PTR;
1619
1620	/* Free the RAM range. */
1621	int rc2 = pgmPhysRamRangeFree(pVM, pRamRange);
1622	AssertLogRelMsgStmt(RT_SUCCESS(rc2), ("rc=%Rrc idx=%u chunk=%u/%u\n", rc, idx, iChunk + 1, cChunks),
1623	rc = RT_SUCCESS(rc) ? rc2 : rc);
1624	}
1625
1626	/*
1627	* Final removal frees up the backing memory.
1628	*/
1629	#ifdef IN_RING3
1630	int const rcBacking = SUPR3PageFree(pbMmio2Backing, cbMmio2Backing >> HOST_PAGE_SHIFT);
1631	AssertLogRelMsgStmt(RT_SUCCESS(rcBacking), ("rc=%Rrc %p LB %#zx\n", rcBacking, pbMmio2Backing, cbMmio2Backing),
1632	rc = RT_SUCCESS(rc) ? rcBacking : rc);
1633	#else
1634	int rcBacking = RTR0MemObjFree(pVM->pgmr0.s.ahMmio2MapObjs[idxFirst], true /fFreeMappings/);
1635	AssertLogRelMsgStmt(RT_SUCCESS(rcBacking),
1636	("rc=%Rrc ahMmio2MapObjs[%u]=%p\n", rcBacking, pVM->pgmr0.s.ahMmio2MapObjs[idxFirst], idxFirst),
1637	rc = RT_SUCCESS(rc) ? rcBacking : rc);
1638	if (RT_SUCCESS(rcBacking))
1639	{
1640	pVM->pgmr0.s.ahMmio2MapObjs[idxFirst] = NIL_RTR0MEMOBJ;
1641
1642	rcBacking = RTR0MemObjFree(pVM->pgmr0.s.ahMmio2MemObjs[idxFirst], true /fFreeMappings/);
1643	AssertLogRelMsgStmt(RT_SUCCESS(rcBacking),
1644	("rc=%Rrc ahMmio2MemObjs[%u]=%p\n", rcBacking, pVM->pgmr0.s.ahMmio2MemObjs[idxFirst], idxFirst),
1645	rc = RT_SUCCESS(rc) ? rcBacking : rc);
1646	if (RT_SUCCESS(rcBacking))
1647	pVM->pgmr0.s.ahMmio2MemObjs[idxFirst] = NIL_RTR0MEMOBJ;
1648	}
1649	#endif
1650
1651	/*
1652	* Decrease the MMIO2 count if these were the last ones.
1653	*/
1654	if (idxFirst + cChunks == pVM->pgm.s.cMmio2Ranges)
1655	pVM->pgm.s.cMmio2Ranges = idxFirst;
1656
1657	/*
1658	* Update page count stats.
1659	*/
1660	pVM->pgm.s.cAllPages -= cGuestPages;
1661	pVM->pgm.s.cPrivatePages -= cGuestPages;
1662
1663	return rc;
1664	}
1665
1666
1667	#ifdef IN_RING0
1668	/**
1669	* This is called during VM state loading to deregister an obsolete MMIO2 range.
1670	*
1671	* This does everything except TLB flushing and releasing the access handler.
1672	* The ranges must be unmapped and wihtout preallocated access handlers.
1673	*
1674	* @returns VBox status code.
1675	* @param pGVM Pointer to the global VM structure.
1676	* @param pReq Where to get the parameters.
1677	* @thread EMT(0)
1678	*/
1679	VMMR0_INT_DECL(int) PGMR0PhysMmio2DeregisterReq(PGVM pGVM, PPGMPHYSMMIO2DEREGISTERREQ pReq)
1680	{
1681	/*
1682	* Validate input (ASSUME pReq is a copy and can't be modified by ring-3
1683	* while we're here).
1684	*/
1685	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
1686	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x < %#zx\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
1687
1688	/** @todo better VM state guard, enmVMState is ring-3 writable. */
1689	/* Only LOADING, as this is special purpose for removing an unwanted PCNet MMIO2 region. */
1690	VMSTATE const enmState = pGVM->enmVMState;
1691	AssertMsgReturn(enmState == VMSTATE_LOADING, ("enmState=%d\n", enmState), VERR_VM_INVALID_VM_STATE);
1692	VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
1693
1694	AssertMsgReturn( pReq->cChunks > 0
1695	&& pReq->cChunks < PGM_MAX_MMIO2_RANGES,
1696	("idMmio2=%#x cChunks=%#x\n", pReq->idMmio2, pReq->cChunks),
1697	VERR_INVALID_PARAMETER);
1698
1699	AssertMsgReturn( pReq->idMmio2 != 0
1700	&& pReq->idMmio2 <= PGM_MAX_MMIO2_RANGES
1701	&& (unsigned)pReq->idMmio2 + pReq->cChunks - 1U <= PGM_MAX_MMIO2_RANGES,
1702	("idMmio2=%#x cChunks=%#x\n", pReq->idMmio2, pReq->cChunks),
1703	VERR_INVALID_PARAMETER);
1704
1705	/*
1706	* Validate that the requested range is for exactly one MMIO2 registration.
1707	*
1708	* This is safe to do w/o the lock because registration and deregistration
1709	* is restricted to EMT0, and we're on EMT0 so can't race ourselves.
1710	*/
1711
1712	/* Check that the first entry is valid and has a memory object for the backing memory. */
1713	uint32_t idx = pReq->idMmio2 - 1;
1714	AssertReturn(pGVM->pgmr0.s.apMmio2RamRanges[idx] != NULL, VERR_INVALID_STATE);
1715	AssertReturn(pGVM->pgmr0.s.ahMmio2MemObjs[idx] != NIL_RTR0MEMOBJ, VERR_INVALID_STATE);
1716
1717	/* Any additional regions must also have RAM ranges, but shall not have any backing memory. */
1718	idx++;
1719	for (uint32_t iChunk = 1; iChunk < pReq->cChunks; iChunk++, idx++)
1720	{
1721	AssertReturn(pGVM->pgmr0.s.apMmio2RamRanges[idx] != NULL, VERR_INVALID_STATE);
1722	AssertReturn(pGVM->pgmr0.s.ahMmio2MemObjs[idx] == NIL_RTR0MEMOBJ, VERR_INVALID_STATE);
1723	}
1724
1725	/* Check that the next entry is for a different region. */
1726	AssertReturn( idx >= RT_ELEMENTS(pGVM->pgmr0.s.apMmio2RamRanges)
1727	\|\| pGVM->pgmr0.s.apMmio2RamRanges[idx] == NULL
1728	\|\| pGVM->pgmr0.s.ahMmio2MemObjs[idx] != NIL_RTR0MEMOBJ,
1729	VERR_INVALID_PARAMETER);
1730
1731	/*
1732	* Make sure we're owning the PGM lock (caller should be) and call the
1733	* common worker code.
1734	*/
1735	int rc = PGM_LOCK(pGVM);
1736	AssertRCReturn(rc, rc);
1737
1738	rc = pgmPhysMmio2DeregisterWorker(pGVM, pReq->idMmio2, pReq->cChunks, pReq->pDevIns);
1739
1740	PGM_UNLOCK(pGVM);
1741	return rc;
1742	}
1743	#endif /* IN_RING0 */
1744
1745
1746
1747
1748	/*********************************************************************************************************************************
1749	* ROM *
1750	*********************************************************************************************************************************/
1751
1752
1753	/**
1754	* Common worker for pgmR3PhysRomRegisterLocked and
1755	* PGMR0PhysRomAllocateRangeReq.
1756	*/
1757	DECLHIDDEN(int) pgmPhysRomRangeAllocCommon(PVMCC pVM, uint32_t cPages, uint8_t idRomRange, uint32_t fFlags)
1758	{
1759	/*
1760	* Allocate the ROM range structure and map it into ring-3.
1761	*/
1762	size_t const cbRomRange = RT_ALIGN_Z(RT_UOFFSETOF_DYN(PGMROMRANGE, aPages[cPages]), HOST_PAGE_SIZE);
1763	#ifdef IN_RING0
1764	RTR0MEMOBJ hMemObj = NIL_RTR0MEMOBJ;
1765	int rc = RTR0MemObjAllocPage(&hMemObj, cbRomRange, false /fExecutable/);
1766	#else
1767	PPGMROMRANGE pRomRange;
1768	int rc = SUPR3PageAlloc(cbRomRange >> HOST_PAGE_SHIFT, 0 /fFlags/, (void **)&pRomRange);
1769	#endif
1770	if (RT_SUCCESS(rc))
1771	{
1772	/* Zero the memory and do basic range init before mapping it into userland. */
1773	#ifdef IN_RING0
1774	PPGMROMRANGE const pRomRange = (PPGMROMRANGE)RTR0MemObjAddress(hMemObj);
1775	if (!RTR0MemObjWasZeroInitialized(hMemObj))
1776	#endif
1777	RT_BZERO(pRomRange, cbRomRange);
1778
1779	pRomRange->GCPhys = NIL_RTGCPHYS;
1780	pRomRange->GCPhysLast = NIL_RTGCPHYS;
1781	pRomRange->cb = (RTGCPHYS)cPages << GUEST_PAGE_SHIFT;
1782	pRomRange->fFlags = fFlags;
1783	pRomRange->idSavedState = UINT8_MAX;
1784	pRomRange->idRamRange = UINT16_MAX;
1785	pRomRange->cbOriginal = 0;
1786	pRomRange->pvOriginal = NIL_RTR3PTR;
1787	pRomRange->pszDesc = NIL_RTR3PTR;
1788
1789	#ifdef IN_RING0
1790	/* Map it into userland. */
1791	RTR0MEMOBJ hMapObj = NIL_RTR0MEMOBJ;
1792	rc = RTR0MemObjMapUser(&hMapObj, hMemObj, (RTR3PTR)-1, 0 /uAlignment/,
1793	RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
1794	if (RT_SUCCESS(rc))
1795	#endif
1796	{
1797	/*
1798	* Grab the lock (unlikely to fail or block as caller typically owns it already).
1799	*/
1800	rc = PGM_LOCK(pVM);
1801	if (RT_SUCCESS(rc))
1802	{
1803	/*
1804	* Check that idRomRange is still free.
1805	*/
1806	if (idRomRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRomRanges))
1807	{
1808	#ifdef IN_RING0
1809	if (pVM->pgmr0.s.apRomRanges[idRomRange] == NULL)
1810	#endif
1811	{
1812	if ( pVM->pgm.s.apRomRanges[idRomRange] == NIL_RTR3PTR
1813	&& pVM->pgm.s.cRomRanges == idRomRange)
1814	{
1815	/*
1816	* Commit it.
1817	*/
1818	#ifdef IN_RING0
1819	pVM->pgmr0.s.apRomRanges[idRomRange] = pRomRange;
1820	pVM->pgmr0.s.acRomRangePages[idRomRange] = cPages;
1821	pVM->pgmr0.s.ahRomRangeMemObjs[idRomRange] = hMemObj;
1822	pVM->pgmr0.s.ahRomRangeMapObjs[idRomRange] = hMapObj;
1823	#endif
1824
1825	pVM->pgm.s.cRomRanges = idRomRange + 1;
1826	#ifdef IN_RING0
1827	pVM->pgm.s.apRomRanges[idRomRange] = RTR0MemObjAddressR3(hMapObj);
1828	#else
1829	pVM->pgm.s.apRomRanges[idRomRange] = pRomRange;
1830	#endif
1831
1832	PGM_UNLOCK(pVM);
1833	return VINF_SUCCESS;
1834	}
1835	}
1836
1837	/*
1838	* Bail out.
1839	*/
1840	rc = VERR_INTERNAL_ERROR_5;
1841	}
1842	else
1843	rc = VERR_PGM_TOO_MANY_ROM_RANGES;
1844	PGM_UNLOCK(pVM);
1845	}
1846	#ifdef IN_RING0
1847	RTR0MemObjFree(hMapObj, false /fFreeMappings/);
1848	#endif
1849	}
1850	#ifdef IN_RING0
1851	RTR0MemObjFree(hMemObj, true /fFreeMappings/);
1852	#else
1853	SUPR3PageFree(pRomRange, cbRomRange >> HOST_PAGE_SHIFT);
1854	#endif
1855	}
1856	return rc;
1857	}
1858
1859
1860	#ifdef IN_RING0
1861	/**
1862	* This is called during VM initialization to allocate a ROM range.
1863	*
1864	* The page array is zeroed, the rest is initialized as best we can based on the
1865	* information in @a pReq.
1866	*
1867	* @returns VBox status code.
1868	* @param pGVM Pointer to the global VM structure.
1869	* @param pReq Where to get the parameters and return the range ID.
1870	* @thread EMT(0)
1871	*/
1872	VMMR0_INT_DECL(int) PGMR0PhysRomAllocateRangeReq(PGVM pGVM, PPGMPHYSROMALLOCATERANGEREQ pReq)
1873	{
1874	/*
1875	* Validate input (ASSUME pReq is a copy and can't be modified by ring-3
1876	* while we're here).
1877	*/
1878	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
1879	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x < %#zx\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
1880
1881	AssertReturn(pReq->cbGuestPage == GUEST_PAGE_SIZE, VERR_INCOMPATIBLE_CONFIG);
1882
1883	AssertReturn(pReq->cGuestPages > 0, VERR_OUT_OF_RANGE);
1884	AssertReturn(pReq->cGuestPages <= PGM_MAX_PAGES_PER_ROM_RANGE, VERR_OUT_OF_RANGE);
1885
1886	AssertMsgReturn(!(pReq->fFlags & ~(uint32_t)PGMPHYS_ROM_FLAGS_VALID_MASK), ("fFlags=%#RX32\n", pReq->fFlags),
1887	VERR_INVALID_FLAGS);
1888
1889	AssertReturn(pReq->idRomRange < RT_ELEMENTS(pGVM->pgmr0.s.apRomRanges), VERR_OUT_OF_RANGE);
1890	AssertReturn(pReq->idRomRange == pGVM->pgm.s.cRomRanges, VERR_OUT_OF_RANGE);
1891
1892	/** @todo better VM state guard, enmVMState is ring-3 writable. */
1893	VMSTATE const enmState = pGVM->enmVMState;
1894	AssertMsgReturn(enmState == VMSTATE_CREATING, ("enmState=%d\n", enmState), VERR_VM_INVALID_VM_STATE);
1895	VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
1896
1897	/*
1898	* Call common worker.
1899	*/
1900	return pgmPhysRomRangeAllocCommon(pGVM, pReq->cGuestPages, pReq->idRomRange, pReq->fFlags);
1901	}
1902	#endif /* IN_RING0 */
1903
1904
1905	/*********************************************************************************************************************************
1906	* Other stuff
1907	*********************************************************************************************************************************/
1908
1909
1910
1911	/**
1912	* Checks if Address Gate 20 is enabled or not.
1913	*
1914	* @returns true if enabled.
1915	* @returns false if disabled.
1916	* @param pVCpu The cross context virtual CPU structure.
1917	*/
1918	VMMDECL(bool) PGMPhysIsA20Enabled(PVMCPU pVCpu)
1919	{
1920	/* Must check that pVCpu isn't NULL here because PDM device helper are a little lazy. */
1921	LogFlow(("PGMPhysIsA20Enabled %d\n", pVCpu && pVCpu->pgm.s.fA20Enabled));
1922	return pVCpu && pVCpu->pgm.s.fA20Enabled;
1923	}
1924
1925
1926	/**
1927	* Validates a GC physical address.
1928	*
1929	* @returns true if valid.
1930	* @returns false if invalid.
1931	* @param pVM The cross context VM structure.
1932	* @param GCPhys The physical address to validate.
1933	*/
1934	VMMDECL(bool) PGMPhysIsGCPhysValid(PVMCC pVM, RTGCPHYS GCPhys)
1935	{
1936	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
1937	return pPage != NULL;
1938	}
1939
1940
1941	/**
1942	* Checks if a GC physical address is a normal page,
1943	* i.e. not ROM, MMIO or reserved.
1944	*
1945	* @returns true if normal.
1946	* @returns false if invalid, ROM, MMIO or reserved page.
1947	* @param pVM The cross context VM structure.
1948	* @param GCPhys The physical address to check.
1949	*/
1950	VMMDECL(bool) PGMPhysIsGCPhysNormal(PVMCC pVM, RTGCPHYS GCPhys)
1951	{
1952	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
1953	return pPage
1954	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM;
1955	}
1956
1957
1958	/**
1959	* Converts a GC physical address to a HC physical address.
1960	*
1961	* @returns VINF_SUCCESS on success.
1962	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
1963	* page but has no physical backing.
1964	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
1965	* GC physical address.
1966	*
1967	* @param pVM The cross context VM structure.
1968	* @param GCPhys The GC physical address to convert.
1969	* @param pHCPhys Where to store the HC physical address on success.
1970	*/
1971	VMM_INT_DECL(int) PGMPhysGCPhys2HCPhys(PVMCC pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
1972	{
1973	PGM_LOCK_VOID(pVM);
1974	PPGMPAGE pPage;
1975	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1976	if (RT_SUCCESS(rc))
1977	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & GUEST_PAGE_OFFSET_MASK);
1978	PGM_UNLOCK(pVM);
1979	return rc;
1980	}
1981
1982
1983	/**
1984	* Invalidates all page mapping TLBs.
1985	*
1986	* @param pVM The cross context VM structure.
1987	* @param fInRendezvous Set if we're in a rendezvous.
1988	*/
1989	void pgmPhysInvalidatePageMapTLB(PVMCC pVM, bool fInRendezvous)
1990	{
1991	PGM_LOCK_VOID(pVM);
1992	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatPageMapTlbFlushes);
1993
1994	/* Clear the R3 & R0 TLBs completely. */
1995	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR0.aEntries); i++)
1996	{
1997	pVM->pgm.s.PhysTlbR0.aEntries[i].GCPhys = NIL_RTGCPHYS;
1998	pVM->pgm.s.PhysTlbR0.aEntries[i].pPage = 0;
1999	pVM->pgm.s.PhysTlbR0.aEntries[i].pv = 0;
2000	}
2001
2002	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR3.aEntries); i++)
2003	{
2004	pVM->pgm.s.PhysTlbR3.aEntries[i].GCPhys = NIL_RTGCPHYS;
2005	pVM->pgm.s.PhysTlbR3.aEntries[i].pPage = 0;
2006	pVM->pgm.s.PhysTlbR3.aEntries[i].pMap = 0;
2007	pVM->pgm.s.PhysTlbR3.aEntries[i].pv = 0;
2008	}
2009
2010	/* For the per VCPU lockless TLBs, we only invalid the GCPhys members so that
2011	anyone concurrently using the entry can safely continue to do so while any
2012	subsequent attempts to use it will fail. (Emulating a scenario where we
2013	lost the PGM lock race and the concurrent TLB user wont it.) */
2014	VMCC_FOR_EACH_VMCPU(pVM)
2015	{
2016	if (!fInRendezvous && pVCpu != VMMGetCpu(pVM))
2017	for (unsigned idx = 0; idx < RT_ELEMENTS(pVCpu->pgm.s.PhysTlb.aEntries); idx++)
2018	ASMAtomicWriteU64(&pVCpu->pgm.s.PhysTlb.aEntries[idx].GCPhys, NIL_RTGCPHYS);
2019	else
2020	for (unsigned idx = 0; idx < RT_ELEMENTS(pVCpu->pgm.s.PhysTlb.aEntries); idx++)
2021	pVCpu->pgm.s.PhysTlb.aEntries[idx].GCPhys = NIL_RTGCPHYS;
2022	}
2023	VMCC_FOR_EACH_VMCPU_END(pVM);
2024
2025	IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID, IEMTLBPHYSFLUSHREASON_MISC);
2026	PGM_UNLOCK(pVM);
2027	}
2028
2029
2030	/**
2031	* Invalidates a page mapping TLB entry
2032	*
2033	* @param pVM The cross context VM structure.
2034	* @param GCPhys GCPhys entry to flush
2035	*
2036	* @note Caller is responsible for calling IEMTlbInvalidateAllPhysicalAllCpus
2037	* when needed.
2038	*/
2039	void pgmPhysInvalidatePageMapTLBEntry(PVMCC pVM, RTGCPHYS GCPhys)
2040	{
2041	PGM_LOCK_ASSERT_OWNER(pVM);
2042
2043	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatPageMapTlbFlushEntry);
2044
2045	unsigned const idx = PGM_PAGER3MAPTLB_IDX(GCPhys);
2046
2047	pVM->pgm.s.PhysTlbR0.aEntries[idx].GCPhys = NIL_RTGCPHYS;
2048	pVM->pgm.s.PhysTlbR0.aEntries[idx].pPage = 0;
2049	pVM->pgm.s.PhysTlbR0.aEntries[idx].pv = 0;
2050
2051	pVM->pgm.s.PhysTlbR3.aEntries[idx].GCPhys = NIL_RTGCPHYS;
2052	pVM->pgm.s.PhysTlbR3.aEntries[idx].pPage = 0;
2053	pVM->pgm.s.PhysTlbR3.aEntries[idx].pMap = 0;
2054	pVM->pgm.s.PhysTlbR3.aEntries[idx].pv = 0;
2055
2056	/* For the per VCPU lockless TLBs, we only invalid the GCPhys member so that
2057	anyone concurrently using the entry can safely continue to do so while any
2058	subsequent attempts to use it will fail. (Emulating a scenario where we
2059	lost the PGM lock race and the concurrent TLB user wont it.) */
2060	VMCC_FOR_EACH_VMCPU(pVM)
2061	{
2062	ASMAtomicWriteU64(&pVCpu->pgm.s.PhysTlb.aEntries[idx].GCPhys, NIL_RTGCPHYS);
2063	}
2064	VMCC_FOR_EACH_VMCPU_END(pVM);
2065	}
2066
2067	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2068
2069	/**
2070	* Makes sure that there is at least one handy page ready for use.
2071	*
2072	* This will also take the appropriate actions when reaching water-marks.
2073	*
2074	* @returns VBox status code.
2075	* @retval VINF_SUCCESS on success.
2076	* @retval VERR_EM_NO_MEMORY if we're really out of memory.
2077	*
2078	* @param pVM The cross context VM structure.
2079	*
2080	* @remarks Must be called from within the PGM critical section. It may
2081	* nip back to ring-3/0 in some cases.
2082	*/
2083	static int pgmPhysEnsureHandyPage(PVMCC pVM)
2084	{
2085	AssertMsg(pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", pVM->pgm.s.cHandyPages));
2086
2087	/*
2088	* Do we need to do anything special?
2089	*/
2090	# ifdef IN_RING3
2091	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_R3_ALLOC))
2092	# else
2093	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_RZ_TO_R3))
2094	# endif
2095	{
2096	/*
2097	* Allocate pages only if we're out of them, or in ring-3, almost out.
2098	*/
2099	# ifdef IN_RING3
2100	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_R3_ALLOC)
2101	# else
2102	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_ALLOC)
2103	# endif
2104	{
2105	Log(("PGM: cHandyPages=%u out of %u -> allocate more; VM_FF_PGM_NO_MEMORY=%RTbool\n",
2106	pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages), VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY) ));
2107	# ifdef IN_RING3
2108	int rc = PGMR3PhysAllocateHandyPages(pVM);
2109	# else
2110	int rc = pgmR0PhysAllocateHandyPages(pVM, VMMGetCpuId(pVM), false /fRing3/);
2111	# endif
2112	if (RT_UNLIKELY(rc != VINF_SUCCESS))
2113	{
2114	if (RT_FAILURE(rc))
2115	return rc;
2116	AssertMsgReturn(rc == VINF_EM_NO_MEMORY, ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
2117	if (!pVM->pgm.s.cHandyPages)
2118	{
2119	LogRel(("PGM: no more handy pages!\n"));
2120	return VERR_EM_NO_MEMORY;
2121	}
2122	Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES));
2123	Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY));
2124	# ifndef IN_RING3
2125	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); /* paranoia */
2126	# endif
2127	}
2128	AssertMsgReturn( pVM->pgm.s.cHandyPages > 0
2129	&& pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages),
2130	("%u\n", pVM->pgm.s.cHandyPages),
2131	VERR_PGM_HANDY_PAGE_IPE);
2132	}
2133	else
2134	{
2135	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_SET_FF)
2136	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
2137	# ifndef IN_RING3
2138	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_TO_R3)
2139	{
2140	Log(("PGM: VM_FF_TO_R3 - cHandyPages=%u out of %u\n", pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
2141	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3);
2142	}
2143	# endif
2144	}
2145	}
2146
2147	return VINF_SUCCESS;
2148	}
2149
2150
2151	/**
2152	* Replace a zero or shared page with new page that we can write to.
2153	*
2154	* @returns The following VBox status codes.
2155	* @retval VINF_SUCCESS on success, pPage is modified.
2156	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
2157	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
2158	*
2159	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
2160	*
2161	* @param pVM The cross context VM structure.
2162	* @param pPage The physical page tracking structure. This will
2163	* be modified on success.
2164	* @param GCPhys The address of the page.
2165	*
2166	* @remarks Must be called from within the PGM critical section. It may
2167	* nip back to ring-3/0 in some cases.
2168	*
2169	* @remarks This function shouldn't really fail, however if it does
2170	* it probably means we've screwed up the size of handy pages and/or
2171	* the low-water mark. Or, that some device I/O is causing a lot of
2172	* pages to be allocated while while the host is in a low-memory
2173	* condition. This latter should be handled elsewhere and in a more
2174	* controlled manner, it's on the @bugref{3170} todo list...
2175	*/
2176	int pgmPhysAllocPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
2177	{
2178	LogFlow(("pgmPhysAllocPage: %R[pgmpage] %RGp\n", pPage, GCPhys));
2179
2180	/*
2181	* Prereqs.
2182	*/
2183	PGM_LOCK_ASSERT_OWNER(pVM);
2184	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
2185	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
2186
2187	# ifdef PGM_WITH_LARGE_PAGES
2188	/*
2189	* Try allocate a large page if applicable.
2190	*/
2191	if ( PGMIsUsingLargePages(pVM)
2192	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2193	&& !VM_IS_NEM_ENABLED(pVM)) /** @todo NEM: Implement large pages support. */
2194	{
2195	RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
2196	PPGMPAGE pBasePage;
2197
2198	int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pBasePage);
2199	AssertRCReturn(rc, rc); /* paranoia; can't happen. */
2200	if (PGM_PAGE_GET_PDE_TYPE(pBasePage) == PGM_PAGE_PDE_TYPE_DONTCARE)
2201	{
2202	rc = pgmPhysAllocLargePage(pVM, GCPhys);
2203	if (rc == VINF_SUCCESS)
2204	return rc;
2205	}
2206	/* Mark the base as type page table, so we don't check over and over again. */
2207	PGM_PAGE_SET_PDE_TYPE(pVM, pBasePage, PGM_PAGE_PDE_TYPE_PT);
2208
2209	/* fall back to 4KB pages. */
2210	}
2211	# endif
2212
2213	/*
2214	* Flush any shadow page table mappings of the page.
2215	* When VBOX_WITH_NEW_LAZY_PAGE_ALLOC isn't defined, there shouldn't be any.
2216	*/
2217	bool fFlushTLBs = false;
2218	int rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhys, pPage, true /fFlushTLBs/, &fFlushTLBs);
2219	AssertMsgReturn(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc), RT_FAILURE(rc) ? rc : VERR_IPE_UNEXPECTED_STATUS);
2220
2221	/*
2222	* Ensure that we've got a page handy, take it and use it.
2223	*/
2224	int rc2 = pgmPhysEnsureHandyPage(pVM);
2225	if (RT_FAILURE(rc2))
2226	{
2227	if (fFlushTLBs)
2228	PGM_INVL_ALL_VCPU_TLBS(pVM);
2229	Assert(rc2 == VERR_EM_NO_MEMORY);
2230	return rc2;
2231	}
2232	/* re-assert preconditions since pgmPhysEnsureHandyPage may do a context switch. */
2233	PGM_LOCK_ASSERT_OWNER(pVM);
2234	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
2235	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
2236
2237	uint32_t iHandyPage = --pVM->pgm.s.cHandyPages;
2238	AssertMsg(iHandyPage < RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", iHandyPage));
2239	Assert(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys != NIL_GMMPAGEDESC_PHYS);
2240	Assert(!(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
2241	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idPage != NIL_GMM_PAGEID);
2242	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID);
2243
2244	/*
2245	* There are one or two action to be taken the next time we allocate handy pages:
2246	* - Tell the GMM (global memory manager) what the page is being used for.
2247	* (Speeds up replacement operations - sharing and defragmenting.)
2248	* - If the current backing is shared, it must be freed.
2249	*/
2250	const RTHCPHYS HCPhys = pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys;
2251	pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys = GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
2252
2253	void const *pvSharedPage = NULL;
2254	if (!PGM_PAGE_IS_SHARED(pPage))
2255	{
2256	Log2(("PGM: Replaced zero page %RGp with %#x / %RHp\n", GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
2257	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatRZPageReplaceZero);
2258	pVM->pgm.s.cZeroPages--;
2259	}
2260	else
2261	{
2262	/* Mark this shared page for freeing/dereferencing. */
2263	pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage = PGM_PAGE_GET_PAGEID(pPage);
2264	Assert(PGM_PAGE_GET_PAGEID(pPage) != NIL_GMM_PAGEID);
2265
2266	Log(("PGM: Replaced shared page %#x at %RGp with %#x / %RHp\n", PGM_PAGE_GET_PAGEID(pPage),
2267	GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
2268	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageReplaceShared));
2269	pVM->pgm.s.cSharedPages--;
2270
2271	/* Grab the address of the page so we can make a copy later on. (safe) */
2272	rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvSharedPage);
2273	AssertRC(rc);
2274	}
2275
2276	/*
2277	* Do the PGMPAGE modifications.
2278	*/
2279	pVM->pgm.s.cPrivatePages++;
2280	PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
2281	PGM_PAGE_SET_PAGEID(pVM, pPage, pVM->pgm.s.aHandyPages[iHandyPage].idPage);
2282	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
2283	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PT);
2284	pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
2285	IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID,
2286	!pvSharedPage
2287	? IEMTLBPHYSFLUSHREASON_ALLOCATED : IEMTLBPHYSFLUSHREASON_ALLOCATED_FROM_SHARED);
2288
2289	/* Copy the shared page contents to the replacement page. */
2290	if (!pvSharedPage)
2291	{ /* likely */ }
2292	else
2293	{
2294	/* Get the virtual address of the new page. */
2295	PGMPAGEMAPLOCK PgMpLck;
2296	void *pvNewPage;
2297	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvNewPage, &PgMpLck); AssertRC(rc);
2298	if (RT_SUCCESS(rc))
2299	{
2300	memcpy(pvNewPage, pvSharedPage, GUEST_PAGE_SIZE); /** @todo todo write ASMMemCopyPage */
2301	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2302	}
2303	}
2304
2305	if ( fFlushTLBs
2306	&& rc != VINF_PGM_GCPHYS_ALIASED)
2307	PGM_INVL_ALL_VCPU_TLBS(pVM);
2308
2309	/*
2310	* Notify NEM about the mapping change for this page.
2311	*
2312	* Note! Shadow ROM pages are complicated as they can definitely be
2313	* allocated while not visible, so play safe.
2314	*/
2315	if (VM_IS_NEM_ENABLED(pVM))
2316	{
2317	PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
2318	if ( enmType != PGMPAGETYPE_ROM_SHADOW
2319	\|\| pgmPhysGetPage(pVM, GCPhys) == pPage)
2320	{
2321	uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
2322	rc2 = NEMHCNotifyPhysPageAllocated(pVM, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, HCPhys,
2323	pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
2324	if (RT_SUCCESS(rc))
2325	PGM_PAGE_SET_NEM_STATE(pPage, u2State);
2326	else
2327	rc = rc2;
2328	}
2329	}
2330
2331	return rc;
2332	}
2333
2334	# ifdef PGM_WITH_LARGE_PAGES
2335
2336	/**
2337	* Replace a 2 MB range of zero pages with new pages that we can write to.
2338	*
2339	* @returns The following VBox status codes.
2340	* @retval VINF_SUCCESS on success, pPage is modified.
2341	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
2342	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
2343	*
2344	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
2345	*
2346	* @param pVM The cross context VM structure.
2347	* @param GCPhys The address of the page.
2348	*
2349	* @remarks Must be called from within the PGM critical section. It may block
2350	* on GMM and host mutexes/locks, leaving HM context.
2351	*/
2352	int pgmPhysAllocLargePage(PVMCC pVM, RTGCPHYS GCPhys)
2353	{
2354	RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
2355	LogFlow(("pgmPhysAllocLargePage: %RGp base %RGp\n", GCPhys, GCPhysBase));
2356	Assert(!VM_IS_NEM_ENABLED(pVM)); /** @todo NEM: Large page support. */
2357
2358	/*
2359	* Check Prereqs.
2360	*/
2361	PGM_LOCK_ASSERT_OWNER(pVM);
2362	Assert(PGMIsUsingLargePages(pVM));
2363
2364	/*
2365	* All the pages must be unallocated RAM pages, i.e. mapping the ZERO page.
2366	*/
2367	PPGMPAGE pFirstPage;
2368	int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pFirstPage);
2369	if ( RT_SUCCESS(rc)
2370	&& PGM_PAGE_GET_TYPE(pFirstPage) == PGMPAGETYPE_RAM
2371	&& PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ZERO)
2372	{
2373	/*
2374	* Further they should have PDE type set to PGM_PAGE_PDE_TYPE_DONTCARE,
2375	* since they are unallocated.
2376	*/
2377	unsigned uPDEType = PGM_PAGE_GET_PDE_TYPE(pFirstPage);
2378	Assert(uPDEType != PGM_PAGE_PDE_TYPE_PDE);
2379	if (uPDEType == PGM_PAGE_PDE_TYPE_DONTCARE)
2380	{
2381	/*
2382	* Now, make sure all the other pages in the 2 MB is in the same state.
2383	*/
2384	GCPhys = GCPhysBase;
2385	unsigned cLeft = _2M / GUEST_PAGE_SIZE;
2386	while (cLeft-- > 0)
2387	{
2388	PPGMPAGE pSubPage = pgmPhysGetPage(pVM, GCPhys);
2389	if ( pSubPage
2390	&& PGM_PAGE_GET_TYPE(pSubPage) == PGMPAGETYPE_RAM /* Anything other than ram implies monitoring. */
2391	&& PGM_PAGE_GET_STATE(pSubPage) == PGM_PAGE_STATE_ZERO) /* Allocated, monitored or shared means we can't use a large page here */
2392	{
2393	Assert(PGM_PAGE_GET_PDE_TYPE(pSubPage) == PGM_PAGE_PDE_TYPE_DONTCARE);
2394	GCPhys += GUEST_PAGE_SIZE;
2395	}
2396	else
2397	{
2398	LogFlow(("pgmPhysAllocLargePage: Found page %RGp with wrong attributes (type=%d; state=%d); cancel check.\n",
2399	GCPhys, pSubPage ? PGM_PAGE_GET_TYPE(pSubPage) : -1, pSubPage ? PGM_PAGE_GET_STATE(pSubPage) : -1));
2400
2401	/* Failed. Mark as requiring a PT so we don't check the whole thing again in the future. */
2402	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRefused);
2403	PGM_PAGE_SET_PDE_TYPE(pVM, pFirstPage, PGM_PAGE_PDE_TYPE_PT);
2404	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
2405	}
2406	}
2407
2408	/*
2409	* Do the allocation.
2410	*/
2411	# ifdef IN_RING3
2412	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_PAGE, GCPhysBase, NULL);
2413	# elif defined(IN_RING0)
2414	rc = pgmR0PhysAllocateLargePage(pVM, VMMGetCpuId(pVM), GCPhysBase);
2415	# else
2416	# error "Port me"
2417	# endif
2418	if (RT_SUCCESS(rc))
2419	{
2420	Assert(PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ALLOCATED);
2421	pVM->pgm.s.cLargePages++;
2422	return VINF_SUCCESS;
2423	}
2424
2425	/* If we fail once, it most likely means the host's memory is too
2426	fragmented; don't bother trying again. */
2427	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
2428	return rc;
2429	}
2430	}
2431	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
2432	}
2433
2434
2435	/**
2436	* Recheck the entire 2 MB range to see if we can use it again as a large page.
2437	*
2438	* @returns The following VBox status codes.
2439	* @retval VINF_SUCCESS on success, the large page can be used again
2440	* @retval VERR_PGM_INVALID_LARGE_PAGE_RANGE if it can't be reused
2441	*
2442	* @param pVM The cross context VM structure.
2443	* @param GCPhys The address of the page.
2444	* @param pLargePage Page structure of the base page
2445	*/
2446	int pgmPhysRecheckLargePage(PVMCC pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage)
2447	{
2448	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRecheck);
2449
2450	Assert(!VM_IS_NEM_ENABLED(pVM)); /** @todo NEM: Large page support. */
2451
2452	AssertCompile(X86_PDE2M_PAE_PG_MASK == EPT_PDE2M_PG_MASK); /* Paranoia: Caller uses this for guest EPT tables as well. */
2453	GCPhys &= X86_PDE2M_PAE_PG_MASK;
2454
2455	/* Check the base page. */
2456	Assert(PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED);
2457	if ( PGM_PAGE_GET_STATE(pLargePage) != PGM_PAGE_STATE_ALLOCATED
2458	\|\| PGM_PAGE_GET_TYPE(pLargePage) != PGMPAGETYPE_RAM
2459	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
2460	{
2461	LogFlow(("pgmPhysRecheckLargePage: checks failed for base page %x %x %x\n", PGM_PAGE_GET_STATE(pLargePage), PGM_PAGE_GET_TYPE(pLargePage), PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage)));
2462	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
2463	}
2464
2465	STAM_PROFILE_START(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,IsValidLargePage), a);
2466	/* Check all remaining pages in the 2 MB range. */
2467	unsigned i;
2468	GCPhys += GUEST_PAGE_SIZE;
2469	for (i = 1; i < _2M / GUEST_PAGE_SIZE; i++)
2470	{
2471	PPGMPAGE pPage;
2472	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2473	AssertRCBreak(rc);
2474
2475	if ( PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2476	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
2477	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
2478	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
2479	{
2480	LogFlow(("pgmPhysRecheckLargePage: checks failed for page %d; %x %x %x\n", i, PGM_PAGE_GET_STATE(pPage), PGM_PAGE_GET_TYPE(pPage), PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)));
2481	break;
2482	}
2483
2484	GCPhys += GUEST_PAGE_SIZE;
2485	}
2486	STAM_PROFILE_STOP(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,IsValidLargePage), a);
2487
2488	if (i == _2M / GUEST_PAGE_SIZE)
2489	{
2490	PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE);
2491	pVM->pgm.s.cLargePagesDisabled--;
2492	Log(("pgmPhysRecheckLargePage: page %RGp can be reused!\n", GCPhys - _2M));
2493	return VINF_SUCCESS;
2494	}
2495
2496	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
2497	}
2498
2499	# endif /* PGM_WITH_LARGE_PAGES */
2500	#endif /* !VBOX_WITH_ONLY_PGM_NEM_MODE */
2501
2502
2503
2504	/**
2505	* Deal with a write monitored page.
2506	*
2507	* @param pVM The cross context VM structure.
2508	* @param pPage The physical page tracking structure.
2509	* @param GCPhys The guest physical address of the page.
2510	* PGMPhysReleasePageMappingLock() passes NIL_RTGCPHYS in a
2511	* very unlikely situation where it is okay that we let NEM
2512	* fix the page access in a lazy fasion.
2513	*
2514	* @remarks Called from within the PGM critical section.
2515	*/
2516	void pgmPhysPageMakeWriteMonitoredWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
2517	{
2518	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED);
2519	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
2520	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
2521	if (PGM_PAGE_IS_CODE_PAGE(pPage))
2522	{
2523	PGM_PAGE_CLEAR_CODE_PAGE(pVM, pPage);
2524	IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID, IEMTLBPHYSFLUSHREASON_MADE_WRITABLE);
2525	}
2526
2527	Assert(pVM->pgm.s.cMonitoredPages > 0);
2528	pVM->pgm.s.cMonitoredPages--;
2529	pVM->pgm.s.cWrittenToPages++;
2530
2531	#ifdef VBOX_WITH_NATIVE_NEM
2532	/*
2533	* Notify NEM about the protection change so we won't spin forever.
2534	*
2535	* Note! NEM need to be handle to lazily correct page protection as we cannot
2536	* really get it 100% right here it seems. The page pool does this too.
2537	*/
2538	if (VM_IS_NEM_ENABLED(pVM) && GCPhys != NIL_RTGCPHYS)
2539	{
2540	uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
2541	PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
2542	PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys);
2543	NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage),
2544	pRam ? PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhys) : NULL,
2545	pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
2546	PGM_PAGE_SET_NEM_STATE(pPage, u2State);
2547	}
2548	#else
2549	RT_NOREF(GCPhys);
2550	#endif
2551	}
2552
2553
2554	/**
2555	* Deal with pages that are not writable, i.e. not in the ALLOCATED state.
2556	*
2557	* @returns VBox strict status code.
2558	* @retval VINF_SUCCESS on success.
2559	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
2560	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2561	*
2562	* @param pVM The cross context VM structure.
2563	* @param pPage The physical page tracking structure.
2564	* @param GCPhys The address of the page.
2565	*
2566	* @remarks Called from within the PGM critical section.
2567	*/
2568	int pgmPhysPageMakeWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
2569	{
2570	PGM_LOCK_ASSERT_OWNER(pVM);
2571	switch (PGM_PAGE_GET_STATE(pPage))
2572	{
2573	case PGM_PAGE_STATE_WRITE_MONITORED:
2574	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
2575	RT_FALL_THRU();
2576	default: /* to shut up GCC */
2577	case PGM_PAGE_STATE_ALLOCATED:
2578	return VINF_SUCCESS;
2579
2580	/*
2581	* Zero pages can be dummy pages for MMIO or reserved memory,
2582	* so we need to check the flags before joining cause with
2583	* shared page replacement.
2584	*/
2585	case PGM_PAGE_STATE_ZERO:
2586	if (PGM_PAGE_IS_MMIO(pPage))
2587	return VERR_PGM_PHYS_PAGE_RESERVED;
2588	RT_FALL_THRU();
2589	case PGM_PAGE_STATE_SHARED:
2590	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2591	return pgmPhysAllocPage(pVM, pPage, GCPhys);
2592	#else
2593	AssertFailed(); /** @todo not sure if we make use of ZERO pages or not in NEM-mode, but I can't see how pgmPhysAllocPage would work. */
2594	return VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE;
2595	#endif
2596
2597	/* Not allowed to write to ballooned pages. */
2598	case PGM_PAGE_STATE_BALLOONED:
2599	return VERR_PGM_PHYS_PAGE_BALLOONED;
2600	}
2601	}
2602
2603	#if 0 /* unused */
2604	/**
2605	* Internal usage: Map the page specified by its GMM ID.
2606	*
2607	* This is similar to pgmPhysPageMap
2608	*
2609	* @returns VBox status code.
2610	*
2611	* @param pVM The cross context VM structure.
2612	* @param idPage The Page ID.
2613	* @param HCPhys The physical address (for SUPR0HCPhysToVirt).
2614	* @param ppv Where to store the mapping address.
2615	*
2616	* @remarks Called from within the PGM critical section. The mapping is only
2617	* valid while you are inside this section.
2618	*/
2619	int pgmPhysPageMapByPageID(PVMCC pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv)
2620	{
2621	/*
2622	* Validation.
2623	*/
2624	PGM_LOCK_ASSERT_OWNER(pVM);
2625	AssertReturn(HCPhys && !(HCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2626	const uint32_t idChunk = idPage >> GMM_CHUNKID_SHIFT;
2627	AssertReturn(idChunk != NIL_GMM_CHUNKID, VERR_INVALID_PARAMETER);
2628
2629	#ifdef IN_RING0
2630	# ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
2631	return SUPR0HCPhysToVirt(HCPhys & ~(RTHCPHYS)GUEST_PAGE_OFFSET_MASK, ppv);
2632	# else
2633	return GMMR0PageIdToVirt(pVM, idPage, ppv);
2634	# endif
2635
2636	#else
2637	/*
2638	* Find/make Chunk TLB entry for the mapping chunk.
2639	*/
2640	PPGMCHUNKR3MAP pMap;
2641	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
2642	if (pTlbe->idChunk == idChunk)
2643	{
2644	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbHits));
2645	pMap = pTlbe->pChunk;
2646	}
2647	else
2648	{
2649	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
2650
2651	/*
2652	* Find the chunk, map it if necessary.
2653	*/
2654	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
2655	if (pMap)
2656	pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
2657	else
2658	{
2659	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
2660	if (RT_FAILURE(rc))
2661	return rc;
2662	}
2663
2664	/*
2665	* Enter it into the Chunk TLB.
2666	*/
2667	pTlbe->idChunk = idChunk;
2668	pTlbe->pChunk = pMap;
2669	}
2670
2671	ppv = (uint8_t )pMap->pv + ((idPage & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT);
2672	return VINF_SUCCESS;
2673	#endif
2674	}
2675	#endif /* unused */
2676
2677	/**
2678	* Maps a page into the current virtual address space so it can be accessed.
2679	*
2680	* @returns VBox status code.
2681	* @retval VINF_SUCCESS on success.
2682	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2683	*
2684	* @param pVM The cross context VM structure.
2685	* @param pPage The physical page tracking structure.
2686	* @param GCPhys The address of the page.
2687	* @param ppMap Where to store the address of the mapping tracking structure.
2688	* @param ppv Where to store the mapping address of the page. The page
2689	* offset is masked off!
2690	*
2691	* @remarks Called from within the PGM critical section.
2692	*/
2693	static int pgmPhysPageMapCommon(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv)
2694	{
2695	PGM_LOCK_ASSERT_OWNER(pVM);
2696	NOREF(GCPhys);
2697
2698	/*
2699	* Special cases: MMIO2 and specially aliased MMIO pages.
2700	*/
2701	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2
2702	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO)
2703	{
2704	*ppMap = NULL;
2705
2706	/* Decode the page id to a page in a MMIO2 ram range. */
2707	uint8_t const idMmio2 = PGM_MMIO2_PAGEID_GET_MMIO2_ID(PGM_PAGE_GET_PAGEID(pPage));
2708	uint32_t const iPage = PGM_MMIO2_PAGEID_GET_IDX(PGM_PAGE_GET_PAGEID(pPage));
2709	AssertLogRelMsgReturn((uint8_t)(idMmio2 - 1U) < RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges),
2710	("idMmio2=%u size=%u type=%u GCPHys=%#RGp Id=%u State=%u", idMmio2,
2711	RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges), PGM_PAGE_GET_TYPE(pPage), GCPhys,
2712	pPage->s.idPage, pPage->s.uStateY),
2713	VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2714	PPGMREGMMIO2RANGE const pMmio2Range = &pVM->pgm.s.aMmio2Ranges[idMmio2 - 1];
2715	AssertLogRelReturn(pMmio2Range, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2716	AssertLogRelReturn(pMmio2Range->idMmio2 == idMmio2, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2717	#ifndef IN_RING0
2718	uint32_t const idRamRange = pMmio2Range->idRamRange;
2719	AssertLogRelReturn(idRamRange < RT_ELEMENTS(pVM->pgm.s.apRamRanges), VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2720	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
2721	AssertLogRelReturn(pRamRange, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2722	AssertLogRelReturn(iPage < (pRamRange->cb >> GUEST_PAGE_SHIFT), VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2723	*ppv = pMmio2Range->pbR3 + ((uintptr_t)iPage << GUEST_PAGE_SHIFT);
2724	return VINF_SUCCESS;
2725
2726	#else /* IN_RING0 */
2727	AssertLogRelReturn(iPage < pVM->pgmr0.s.acMmio2RangePages[idMmio2 - 1], VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2728	# ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
2729	return SUPR0HCPhysToVirt(PGM_PAGE_GET_HCPHYS(pPage), ppv);
2730	# else
2731	AssertPtr(pVM->pgmr0.s.apbMmio2Backing[idMmio2 - 1]);
2732	*ppv = pVM->pgmr0.s.apbMmio2Backing[idMmio2 - 1] + ((uintptr_t)iPage << GUEST_PAGE_SHIFT);
2733	return VINF_SUCCESS;
2734	# endif
2735	#endif
2736	}
2737
2738	#ifdef VBOX_WITH_PGM_NEM_MODE
2739	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2740	if (pVM->pgm.s.fNemMode)
2741	# endif
2742	{
2743	# ifdef IN_RING3
2744	/*
2745	* Find the corresponding RAM range and use that to locate the mapping address.
2746	*/
2747	/** @todo Use the page ID for some kind of indexing as we do with MMIO2 above. */
2748	PPGMRAMRANGE const pRam = pgmPhysGetRange(pVM, GCPhys);
2749	AssertLogRelMsgReturn(pRam, ("%RTGp\n", GCPhys), VERR_INTERNAL_ERROR_3);
2750	size_t const idxPage = (GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT;
2751	Assert(pPage == &pRam->aPages[idxPage]);
2752	*ppMap = NULL;
2753	ppv = (uint8_t )pRam->pbR3 + (idxPage << GUEST_PAGE_SHIFT);
2754	return VINF_SUCCESS;
2755	# else
2756	AssertFailedReturn(VERR_INTERNAL_ERROR_2);
2757	# endif
2758	}
2759	#endif /* VBOX_WITH_PGM_NEM_MODE */
2760	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2761
2762	const uint32_t idChunk = PGM_PAGE_GET_CHUNKID(pPage);
2763	if (idChunk == NIL_GMM_CHUNKID)
2764	{
2765	AssertMsgReturn(PGM_PAGE_GET_PAGEID(pPage) == NIL_GMM_PAGEID, ("pPage=%R[pgmpage]\n", pPage),
2766	VERR_PGM_PHYS_PAGE_MAP_IPE_1);
2767	if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
2768	{
2769	AssertMsgReturn(PGM_PAGE_IS_ZERO(pPage), ("pPage=%R[pgmpage]\n", pPage),
2770	VERR_PGM_PHYS_PAGE_MAP_IPE_3);
2771	AssertMsgReturn(PGM_PAGE_GET_HCPHYS(pPage)== pVM->pgm.s.HCPhysZeroPg, ("pPage=%R[pgmpage]\n", pPage),
2772	VERR_PGM_PHYS_PAGE_MAP_IPE_4);
2773	*ppv = pVM->pgm.s.abZeroPg;
2774	}
2775	else
2776	*ppv = pVM->pgm.s.abZeroPg;
2777	*ppMap = NULL;
2778	return VINF_SUCCESS;
2779	}
2780
2781	# if defined(IN_RING0) && defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
2782	/*
2783	* Just use the physical address.
2784	*/
2785	*ppMap = NULL;
2786	return SUPR0HCPhysToVirt(PGM_PAGE_GET_HCPHYS(pPage), ppv);
2787
2788	# elif defined(IN_RING0)
2789	/*
2790	* Go by page ID thru GMMR0.
2791	*/
2792	*ppMap = NULL;
2793	return GMMR0PageIdToVirt(pVM, PGM_PAGE_GET_PAGEID(pPage), ppv);
2794
2795	# else
2796	/*
2797	* Find/make Chunk TLB entry for the mapping chunk.
2798	*/
2799	PPGMCHUNKR3MAP pMap;
2800	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
2801	if (pTlbe->idChunk == idChunk)
2802	{
2803	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbHits));
2804	pMap = pTlbe->pChunk;
2805	AssertPtr(pMap->pv);
2806	}
2807	else
2808	{
2809	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
2810
2811	/*
2812	* Find the chunk, map it if necessary.
2813	*/
2814	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
2815	if (pMap)
2816	{
2817	AssertPtr(pMap->pv);
2818	pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
2819	}
2820	else
2821	{
2822	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
2823	if (RT_FAILURE(rc))
2824	return rc;
2825	AssertPtr(pMap->pv);
2826	}
2827
2828	/*
2829	* Enter it into the Chunk TLB.
2830	*/
2831	pTlbe->idChunk = idChunk;
2832	pTlbe->pChunk = pMap;
2833	}
2834
2835	ppv = (uint8_t )pMap->pv + (PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) << GUEST_PAGE_SHIFT);
2836	*ppMap = pMap;
2837	return VINF_SUCCESS;
2838	# endif /* !IN_RING0 */
2839	#endif /* !VBOX_WITH_ONLY_PGM_NEM_MODE */
2840	}
2841
2842
2843	/**
2844	* Combination of pgmPhysPageMakeWritable and pgmPhysPageMapWritable.
2845	*
2846	* This is typically used is paths where we cannot use the TLB methods (like ROM
2847	* pages) or where there is no point in using them since we won't get many hits.
2848	*
2849	* @returns VBox strict status code.
2850	* @retval VINF_SUCCESS on success.
2851	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
2852	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2853	*
2854	* @param pVM The cross context VM structure.
2855	* @param pPage The physical page tracking structure.
2856	* @param GCPhys The address of the page.
2857	* @param ppv Where to store the mapping address of the page. The page
2858	* offset is masked off!
2859	*
2860	* @remarks Called from within the PGM critical section. The mapping is only
2861	* valid while you are inside section.
2862	*/
2863	int pgmPhysPageMakeWritableAndMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
2864	{
2865	int rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2866	if (RT_SUCCESS(rc))
2867	{
2868	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* returned */, ("%Rrc\n", rc));
2869	PPGMPAGEMAP pMapIgnore;
2870	int rc2 = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
2871	if (RT_FAILURE(rc2)) /* preserve rc */
2872	rc = rc2;
2873	}
2874	return rc;
2875	}
2876
2877
2878	/**
2879	* Maps a page into the current virtual address space so it can be accessed for
2880	* both writing and reading.
2881	*
2882	* This is typically used is paths where we cannot use the TLB methods (like ROM
2883	* pages) or where there is no point in using them since we won't get many hits.
2884	*
2885	* @returns VBox status code.
2886	* @retval VINF_SUCCESS on success.
2887	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2888	*
2889	* @param pVM The cross context VM structure.
2890	* @param pPage The physical page tracking structure. Must be in the
2891	* allocated state.
2892	* @param GCPhys The address of the page.
2893	* @param ppv Where to store the mapping address of the page. The page
2894	* offset is masked off!
2895	*
2896	* @remarks Called from within the PGM critical section. The mapping is only
2897	* valid while you are inside section.
2898	*/
2899	int pgmPhysPageMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
2900	{
2901	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2902	PPGMPAGEMAP pMapIgnore;
2903	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
2904	}
2905
2906
2907	/**
2908	* Maps a page into the current virtual address space so it can be accessed for
2909	* reading.
2910	*
2911	* This is typically used is paths where we cannot use the TLB methods (like ROM
2912	* pages) or where there is no point in using them since we won't get many hits.
2913	*
2914	* @returns VBox status code.
2915	* @retval VINF_SUCCESS on success.
2916	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2917	*
2918	* @param pVM The cross context VM structure.
2919	* @param pPage The physical page tracking structure.
2920	* @param GCPhys The address of the page.
2921	* @param ppv Where to store the mapping address of the page. The page
2922	* offset is masked off!
2923	*
2924	* @remarks Called from within the PGM critical section. The mapping is only
2925	* valid while you are inside this section.
2926	*/
2927	int pgmPhysPageMapReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv)
2928	{
2929	PPGMPAGEMAP pMapIgnore;
2930	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, (void **)ppv);
2931	}
2932
2933
2934	/**
2935	* Load a guest page into the ring-3 physical TLB.
2936	*
2937	* @returns VBox status code.
2938	* @retval VINF_SUCCESS on success
2939	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
2940	* @param pVM The cross context VM structure.
2941	* @param GCPhys The guest physical address in question.
2942	*/
2943	int pgmPhysPageLoadIntoTlb(PVMCC pVM, RTGCPHYS GCPhys)
2944	{
2945	PGM_LOCK_ASSERT_OWNER(pVM);
2946
2947	/*
2948	* Find the ram range and page and hand it over to the with-page function.
2949	* 99.8% of requests are expected to be in the first range.
2950	*/
2951	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
2952	if (!pPage)
2953	{
2954	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageMapTlbMisses));
2955	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
2956	}
2957
2958	return pgmPhysPageLoadIntoTlbWithPage(pVM, pPage, GCPhys);
2959	}
2960
2961
2962	/**
2963	* Load a guest page into the ring-3 physical TLB.
2964	*
2965	* @returns VBox status code.
2966	* @retval VINF_SUCCESS on success
2967	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
2968	*
2969	* @param pVM The cross context VM structure.
2970	* @param pPage Pointer to the PGMPAGE structure corresponding to
2971	* GCPhys.
2972	* @param GCPhys The guest physical address in question.
2973	*/
2974	int pgmPhysPageLoadIntoTlbWithPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
2975	{
2976	PGM_LOCK_ASSERT_OWNER(pVM);
2977	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageMapTlbMisses));
2978
2979	/*
2980	* Map the page.
2981	* Make a special case for the zero page as it is kind of special.
2982	*/
2983	PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTX_SUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
2984	if ( !PGM_PAGE_IS_ZERO(pPage)
2985	&& !PGM_PAGE_IS_BALLOONED(pPage))
2986	{
2987	void *pv;
2988	PPGMPAGEMAP pMap;
2989	int rc = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMap, &pv);
2990	if (RT_FAILURE(rc))
2991	return rc;
2992	#ifndef IN_RING0
2993	pTlbe->pMap = pMap;
2994	#endif
2995	pTlbe->pv = pv;
2996	Assert(!((uintptr_t)pTlbe->pv & GUEST_PAGE_OFFSET_MASK));
2997	}
2998	else
2999	{
3000	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
3001	AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == pVM->pgm.s.HCPhysZeroPg, ("%RGp/%R[pgmpage]\n", GCPhys, pPage));
3002	#endif
3003	#ifndef IN_RING0
3004	pTlbe->pMap = NULL;
3005	#endif
3006	pTlbe->pv = pVM->pgm.s.abZeroPg;
3007	}
3008	#ifdef PGM_WITH_PHYS_TLB
3009	if ( PGM_PAGE_GET_TYPE(pPage) < PGMPAGETYPE_ROM_SHADOW
3010	\|\| PGM_PAGE_GET_TYPE(pPage) > PGMPAGETYPE_ROM)
3011	pTlbe->GCPhys = GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
3012	else
3013	pTlbe->GCPhys = NIL_RTGCPHYS; /* ROM: Problematic because of the two pages. :-/ */
3014	#else
3015	pTlbe->GCPhys = NIL_RTGCPHYS;
3016	#endif
3017	pTlbe->pPage = pPage;
3018	return VINF_SUCCESS;
3019	}
3020
3021
3022	#ifdef IN_RING3 /** @todo Need ensure a ring-0 version gets invalidated safely */
3023	/**
3024	* Load a guest page into the lockless ring-3 physical TLB for the calling EMT.
3025	*
3026	* @returns VBox status code.
3027	* @retval VINF_SUCCESS on success
3028	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3029	*
3030	* @param pVCpu The cross context virtual CPU structure.
3031	* @param pPage Pointer to the PGMPAGE structure corresponding to
3032	* GCPhys.
3033	* @param GCPhys The guest physical address in question.
3034	*/
3035	DECLHIDDEN(int) pgmPhysPageLoadIntoLocklessTlbWithPage(PVMCPUCC pVCpu, PPGMPAGE pPage, RTGCPHYS GCPhys)
3036	{
3037	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageMapTlbMisses));
3038	PPGMPAGEMAPTLBE const pLocklessTlbe = &pVCpu->pgm.s.PhysTlb.aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
3039	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3040
3041	PGM_LOCK_VOID(pVM);
3042
3043	PPGMPAGEMAPTLBE pSharedTlbe;
3044	int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pSharedTlbe);
3045	if (RT_SUCCESS(rc))
3046	pLocklessTlbe = pSharedTlbe;
3047
3048	PGM_UNLOCK(pVM);
3049	return rc;
3050	}
3051	#endif /* IN_RING3 */
3052
3053
3054	/**
3055	* Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
3056	* own the PGM lock and therefore not need to lock the mapped page.
3057	*
3058	* @returns VBox status code.
3059	* @retval VINF_SUCCESS on success.
3060	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3061	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3062	*
3063	* @param pVM The cross context VM structure.
3064	* @param GCPhys The guest physical address of the page that should be mapped.
3065	* @param pPage Pointer to the PGMPAGE structure for the page.
3066	* @param ppv Where to store the address corresponding to GCPhys.
3067	*
3068	* @internal
3069	* @deprecated Use pgmPhysGCPhys2CCPtrInternalEx.
3070	*/
3071	int pgmPhysGCPhys2CCPtrInternalDepr(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
3072	{
3073	int rc;
3074	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
3075	PGM_LOCK_ASSERT_OWNER(pVM);
3076	pVM->pgm.s.cDeprecatedPageLocks++;
3077
3078	/*
3079	* Make sure the page is writable.
3080	*/
3081	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
3082	{
3083	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3084	if (RT_FAILURE(rc))
3085	return rc;
3086	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
3087	}
3088	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0 \|\| PGM_IS_IN_NEM_MODE(pVM));
3089
3090	/*
3091	* Get the mapping address.
3092	*/
3093	PPGMPAGEMAPTLBE pTlbe;
3094	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
3095	if (RT_FAILURE(rc))
3096	return rc;
3097	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3098	return VINF_SUCCESS;
3099	}
3100
3101
3102	/**
3103	* Locks a page mapping for writing.
3104	*
3105	* @param pVM The cross context VM structure.
3106	* @param pPage The page.
3107	* @param pTlbe The mapping TLB entry for the page.
3108	* @param pLock The lock structure (output).
3109	*/
3110	DECLINLINE(void) pgmPhysPageMapLockForWriting(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
3111	{
3112	# ifndef IN_RING0
3113	PPGMPAGEMAP pMap = pTlbe->pMap;
3114	if (pMap)
3115	pMap->cRefs++;
3116	# else
3117	RT_NOREF(pTlbe);
3118	# endif
3119
3120	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
3121	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
3122	{
3123	if (cLocks == 0)
3124	pVM->pgm.s.cWriteLockedPages++;
3125	PGM_PAGE_INC_WRITE_LOCKS(pPage);
3126	}
3127	else if (cLocks != PGM_PAGE_MAX_LOCKS)
3128	{
3129	PGM_PAGE_INC_WRITE_LOCKS(pPage);
3130	AssertMsgFailed(("%R[pgmpage] is entering permanent write locked state!\n", pPage));
3131	# ifndef IN_RING0
3132	if (pMap)
3133	pMap->cRefs++; /* Extra ref to prevent it from going away. */
3134	# endif
3135	}
3136
3137	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
3138	# ifndef IN_RING0
3139	pLock->pvMap = pMap;
3140	# else
3141	pLock->pvMap = NULL;
3142	# endif
3143	}
3144
3145	/**
3146	* Locks a page mapping for reading.
3147	*
3148	* @param pVM The cross context VM structure.
3149	* @param pPage The page.
3150	* @param pTlbe The mapping TLB entry for the page.
3151	* @param pLock The lock structure (output).
3152	*/
3153	DECLINLINE(void) pgmPhysPageMapLockForReading(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
3154	{
3155	# ifndef IN_RING0
3156	PPGMPAGEMAP pMap = pTlbe->pMap;
3157	if (pMap)
3158	pMap->cRefs++;
3159	# else
3160	RT_NOREF(pTlbe);
3161	# endif
3162
3163	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
3164	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
3165	{
3166	if (cLocks == 0)
3167	pVM->pgm.s.cReadLockedPages++;
3168	PGM_PAGE_INC_READ_LOCKS(pPage);
3169	}
3170	else if (cLocks != PGM_PAGE_MAX_LOCKS)
3171	{
3172	PGM_PAGE_INC_READ_LOCKS(pPage);
3173	AssertMsgFailed(("%R[pgmpage] is entering permanent read locked state!\n", pPage));
3174	# ifndef IN_RING0
3175	if (pMap)
3176	pMap->cRefs++; /* Extra ref to prevent it from going away. */
3177	# endif
3178	}
3179
3180	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
3181	# ifndef IN_RING0
3182	pLock->pvMap = pMap;
3183	# else
3184	pLock->pvMap = NULL;
3185	# endif
3186	}
3187
3188
3189	/**
3190	* Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
3191	* own the PGM lock and have access to the page structure.
3192	*
3193	* @returns VBox status code.
3194	* @retval VINF_SUCCESS on success.
3195	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3196	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3197	*
3198	* @param pVM The cross context VM structure.
3199	* @param GCPhys The guest physical address of the page that should be mapped.
3200	* @param pPage Pointer to the PGMPAGE structure for the page.
3201	* @param ppv Where to store the address corresponding to GCPhys.
3202	* @param pLock Where to store the lock information that
3203	* pgmPhysReleaseInternalPageMappingLock needs.
3204	*
3205	* @internal
3206	*/
3207	int pgmPhysGCPhys2CCPtrInternal(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
3208	{
3209	int rc;
3210	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
3211	PGM_LOCK_ASSERT_OWNER(pVM);
3212
3213	/*
3214	* Make sure the page is writable.
3215	*/
3216	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
3217	{
3218	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3219	if (RT_FAILURE(rc))
3220	return rc;
3221	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
3222	}
3223	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0 \|\| PGM_IS_IN_NEM_MODE(pVM));
3224
3225	/*
3226	* Do the job.
3227	*/
3228	PPGMPAGEMAPTLBE pTlbe;
3229	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
3230	if (RT_FAILURE(rc))
3231	return rc;
3232	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
3233	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3234	return VINF_SUCCESS;
3235	}
3236
3237
3238	/**
3239	* Internal version of PGMPhysGCPhys2CCPtrReadOnly that expects the caller to
3240	* own the PGM lock and have access to the page structure.
3241	*
3242	* @returns VBox status code.
3243	* @retval VINF_SUCCESS on success.
3244	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3245	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3246	*
3247	* @param pVM The cross context VM structure.
3248	* @param GCPhys The guest physical address of the page that should be mapped.
3249	* @param pPage Pointer to the PGMPAGE structure for the page.
3250	* @param ppv Where to store the address corresponding to GCPhys.
3251	* @param pLock Where to store the lock information that
3252	* pgmPhysReleaseInternalPageMappingLock needs.
3253	*
3254	* @internal
3255	*/
3256	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv, PPGMPAGEMAPLOCK pLock)
3257	{
3258	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
3259	PGM_LOCK_ASSERT_OWNER(pVM);
3260	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0 \|\| PGM_IS_IN_NEM_MODE(pVM));
3261
3262	/*
3263	* Do the job.
3264	*/
3265	PPGMPAGEMAPTLBE pTlbe;
3266	int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
3267	if (RT_FAILURE(rc))
3268	return rc;
3269	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
3270	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3271	return VINF_SUCCESS;
3272	}
3273
3274
3275	/**
3276	* Requests the mapping of a guest page into the current context.
3277	*
3278	* This API should only be used for very short term, as it will consume scarse
3279	* resources (R0 and GC) in the mapping cache. When you're done with the page,
3280	* call PGMPhysReleasePageMappingLock() ASAP to release it.
3281	*
3282	* This API will assume your intention is to write to the page, and will
3283	* therefore replace shared and zero pages. If you do not intend to modify
3284	* the page, use the PGMPhysGCPhys2CCPtrReadOnly() API.
3285	*
3286	* @returns VBox status code.
3287	* @retval VINF_SUCCESS on success.
3288	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3289	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3290	*
3291	* @param pVM The cross context VM structure.
3292	* @param GCPhys The guest physical address of the page that should be
3293	* mapped.
3294	* @param ppv Where to store the address corresponding to GCPhys.
3295	* @param pLock Where to store the lock information that
3296	* PGMPhysReleasePageMappingLock needs.
3297	*
3298	* @remarks The caller is responsible for dealing with access handlers.
3299	* @todo Add an informational return code for pages with access handlers?
3300	*
3301	* @remark Avoid calling this API from within critical sections (other than
3302	* the PGM one) because of the deadlock risk. External threads may
3303	* need to delegate jobs to the EMTs.
3304	* @remarks Only one page is mapped! Make no assumption about what's after or
3305	* before the returned page!
3306	* @thread Any thread.
3307	*/
3308	VMM_INT_DECL(int) PGMPhysGCPhys2CCPtr(PVMCC pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
3309	{
3310	int rc = PGM_LOCK(pVM);
3311	AssertRCReturn(rc, rc);
3312
3313	/*
3314	* Query the Physical TLB entry for the page (may fail).
3315	*/
3316	PPGMPAGEMAPTLBE pTlbe;
3317	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
3318	if (RT_SUCCESS(rc))
3319	{
3320	/*
3321	* If the page is shared, the zero page, or being write monitored
3322	* it must be converted to a page that's writable if possible.
3323	*/
3324	PPGMPAGE pPage = pTlbe->pPage;
3325	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
3326	{
3327	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3328	if (RT_SUCCESS(rc))
3329	{
3330	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
3331	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
3332	}
3333	}
3334	if (RT_SUCCESS(rc))
3335	{
3336	/*
3337	* Now, just perform the locking and calculate the return address.
3338	*/
3339	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
3340	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3341	}
3342	}
3343
3344	PGM_UNLOCK(pVM);
3345	return rc;
3346	}
3347
3348
3349	/**
3350	* Requests the mapping of a guest page into the current context.
3351	*
3352	* This API should only be used for very short term, as it will consume scarse
3353	* resources (R0 and GC) in the mapping cache. When you're done with the page,
3354	* call PGMPhysReleasePageMappingLock() ASAP to release it.
3355	*
3356	* @returns VBox status code.
3357	* @retval VINF_SUCCESS on success.
3358	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3359	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3360	*
3361	* @param pVM The cross context VM structure.
3362	* @param GCPhys The guest physical address of the page that should be
3363	* mapped.
3364	* @param ppv Where to store the address corresponding to GCPhys.
3365	* @param pLock Where to store the lock information that
3366	* PGMPhysReleasePageMappingLock needs.
3367	*
3368	* @remarks The caller is responsible for dealing with access handlers.
3369	* @todo Add an informational return code for pages with access handlers?
3370	*
3371	* @remarks Avoid calling this API from within critical sections (other than
3372	* the PGM one) because of the deadlock risk.
3373	* @remarks Only one page is mapped! Make no assumption about what's after or
3374	* before the returned page!
3375	* @thread Any thread.
3376	*/
3377	VMM_INT_DECL(int) PGMPhysGCPhys2CCPtrReadOnly(PVMCC pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
3378	{
3379	int rc = PGM_LOCK(pVM);
3380	AssertRCReturn(rc, rc);
3381
3382	/*
3383	* Query the Physical TLB entry for the page (may fail).
3384	*/
3385	PPGMPAGEMAPTLBE pTlbe;
3386	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
3387	if (RT_SUCCESS(rc))
3388	{
3389	/* MMIO pages doesn't have any readable backing. */
3390	PPGMPAGE pPage = pTlbe->pPage;
3391	if (RT_UNLIKELY(PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)))
3392	rc = VERR_PGM_PHYS_PAGE_RESERVED;
3393	else
3394	{
3395	/*
3396	* Now, just perform the locking and calculate the return address.
3397	*/
3398	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
3399	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3400	}
3401	}
3402
3403	PGM_UNLOCK(pVM);
3404	return rc;
3405	}
3406
3407
3408	/**
3409	* Requests the mapping of a guest page given by virtual address into the current context.
3410	*
3411	* This API should only be used for very short term, as it will consume
3412	* scarse resources (R0 and GC) in the mapping cache. When you're done
3413	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
3414	*
3415	* This API will assume your intention is to write to the page, and will
3416	* therefore replace shared and zero pages. If you do not intend to modify
3417	* the page, use the PGMPhysGCPtr2CCPtrReadOnly() API.
3418	*
3419	* @returns VBox status code.
3420	* @retval VINF_SUCCESS on success.
3421	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
3422	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
3423	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3424	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3425	*
3426	* @param pVCpu The cross context virtual CPU structure.
3427	* @param GCPtr The guest physical address of the page that should be
3428	* mapped.
3429	* @param ppv Where to store the address corresponding to GCPhys.
3430	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
3431	*
3432	* @remark Avoid calling this API from within critical sections (other than
3433	* the PGM one) because of the deadlock risk.
3434	* @thread EMT
3435	*/
3436	VMM_INT_DECL(int) PGMPhysGCPtr2CCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, void **ppv, PPGMPAGEMAPLOCK pLock)
3437	{
3438	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
3439	RTGCPHYS GCPhys;
3440	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
3441	if (RT_SUCCESS(rc))
3442	rc = PGMPhysGCPhys2CCPtr(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
3443	return rc;
3444	}
3445
3446
3447	/**
3448	* Requests the mapping of a guest page given by virtual address into the current context.
3449	*
3450	* This API should only be used for very short term, as it will consume
3451	* scarse resources (R0 and GC) in the mapping cache. When you're done
3452	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
3453	*
3454	* @returns VBox status code.
3455	* @retval VINF_SUCCESS on success.
3456	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
3457	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
3458	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3459	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3460	*
3461	* @param pVCpu The cross context virtual CPU structure.
3462	* @param GCPtr The guest physical address of the page that should be
3463	* mapped.
3464	* @param ppv Where to store the address corresponding to GCPtr.
3465	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
3466	*
3467	* @remark Avoid calling this API from within critical sections (other than
3468	* the PGM one) because of the deadlock risk.
3469	* @thread EMT(pVCpu)
3470	*/
3471	VMM_INT_DECL(int) PGMPhysGCPtr2CCPtrReadOnly(PVMCPUCC pVCpu, RTGCPTR GCPtr, void const **ppv, PPGMPAGEMAPLOCK pLock)
3472	{
3473	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
3474	RTGCPHYS GCPhys;
3475	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
3476	if (RT_SUCCESS(rc))
3477	rc = PGMPhysGCPhys2CCPtrReadOnly(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
3478	return rc;
3479	}
3480
3481
3482	/**
3483	* Release the mapping of a guest page.
3484	*
3485	* This is the counter part of PGMPhysGCPhys2CCPtr, PGMPhysGCPhys2CCPtrReadOnly
3486	* PGMPhysGCPtr2CCPtr and PGMPhysGCPtr2CCPtrReadOnly.
3487	*
3488	* @param pVM The cross context VM structure.
3489	* @param pLock The lock structure initialized by the mapping function.
3490	*/
3491	VMMDECL(void) PGMPhysReleasePageMappingLock(PVMCC pVM, PPGMPAGEMAPLOCK pLock)
3492	{
3493	# ifndef IN_RING0
3494	PPGMPAGEMAP pMap = (PPGMPAGEMAP)pLock->pvMap;
3495	# endif
3496	PPGMPAGE pPage = (PPGMPAGE)(pLock->uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
3497	bool fWriteLock = (pLock->uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
3498
3499	pLock->uPageAndType = 0;
3500	pLock->pvMap = NULL;
3501
3502	PGM_LOCK_VOID(pVM);
3503	if (fWriteLock)
3504	{
3505	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
3506	Assert(cLocks > 0);
3507	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
3508	{
3509	if (cLocks == 1)
3510	{
3511	Assert(pVM->pgm.s.cWriteLockedPages > 0);
3512	pVM->pgm.s.cWriteLockedPages--;
3513	}
3514	PGM_PAGE_DEC_WRITE_LOCKS(pPage);
3515	}
3516
3517	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED)
3518	{ /* probably extremely likely */ }
3519	else
3520	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, NIL_RTGCPHYS);
3521	}
3522	else
3523	{
3524	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
3525	Assert(cLocks > 0);
3526	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
3527	{
3528	if (cLocks == 1)
3529	{
3530	Assert(pVM->pgm.s.cReadLockedPages > 0);
3531	pVM->pgm.s.cReadLockedPages--;
3532	}
3533	PGM_PAGE_DEC_READ_LOCKS(pPage);
3534	}
3535	}
3536
3537	# ifndef IN_RING0
3538	if (pMap)
3539	{
3540	Assert(pMap->cRefs >= 1);
3541	pMap->cRefs--;
3542	}
3543	# endif
3544	PGM_UNLOCK(pVM);
3545	}
3546
3547
3548	#ifdef IN_RING3
3549	/**
3550	* Release the mapping of multiple guest pages.
3551	*
3552	* This is the counter part to PGMR3PhysBulkGCPhys2CCPtrExternal() and
3553	* PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal().
3554	*
3555	* @param pVM The cross context VM structure.
3556	* @param cPages Number of pages to unlock.
3557	* @param paLocks Array of locks lock structure initialized by the mapping
3558	* function.
3559	*/
3560	VMMDECL(void) PGMPhysBulkReleasePageMappingLocks(PVMCC pVM, uint32_t cPages, PPGMPAGEMAPLOCK paLocks)
3561	{
3562	Assert(cPages > 0);
3563	bool const fWriteLock = (paLocks[0].uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
3564	#ifdef VBOX_STRICT
3565	for (uint32_t i = 1; i < cPages; i++)
3566	{
3567	Assert(fWriteLock == ((paLocks[i].uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE));
3568	AssertPtr(paLocks[i].uPageAndType);
3569	}
3570	#endif
3571
3572	PGM_LOCK_VOID(pVM);
3573	if (fWriteLock)
3574	{
3575	/*
3576	* Write locks:
3577	*/
3578	for (uint32_t i = 0; i < cPages; i++)
3579	{
3580	PPGMPAGE pPage = (PPGMPAGE)(paLocks[i].uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
3581	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
3582	Assert(cLocks > 0);
3583	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
3584	{
3585	if (cLocks == 1)
3586	{
3587	Assert(pVM->pgm.s.cWriteLockedPages > 0);
3588	pVM->pgm.s.cWriteLockedPages--;
3589	}
3590	PGM_PAGE_DEC_WRITE_LOCKS(pPage);
3591	}
3592
3593	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED)
3594	{ /* probably extremely likely */ }
3595	else
3596	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, NIL_RTGCPHYS);
3597
3598	PPGMPAGEMAP pMap = (PPGMPAGEMAP)paLocks[i].pvMap;
3599	if (pMap)
3600	{
3601	Assert(pMap->cRefs >= 1);
3602	pMap->cRefs--;
3603	}
3604
3605	/* Yield the lock: */
3606	if ((i & 1023) == 1023 && i + 1 < cPages)
3607	{
3608	PGM_UNLOCK(pVM);
3609	PGM_LOCK_VOID(pVM);
3610	}
3611	}
3612	}
3613	else
3614	{
3615	/*
3616	* Read locks:
3617	*/
3618	for (uint32_t i = 0; i < cPages; i++)
3619	{
3620	PPGMPAGE pPage = (PPGMPAGE)(paLocks[i].uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
3621	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
3622	Assert(cLocks > 0);
3623	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
3624	{
3625	if (cLocks == 1)
3626	{
3627	Assert(pVM->pgm.s.cReadLockedPages > 0);
3628	pVM->pgm.s.cReadLockedPages--;
3629	}
3630	PGM_PAGE_DEC_READ_LOCKS(pPage);
3631	}
3632
3633	PPGMPAGEMAP pMap = (PPGMPAGEMAP)paLocks[i].pvMap;
3634	if (pMap)
3635	{
3636	Assert(pMap->cRefs >= 1);
3637	pMap->cRefs--;
3638	}
3639
3640	/* Yield the lock: */
3641	if ((i & 1023) == 1023 && i + 1 < cPages)
3642	{
3643	PGM_UNLOCK(pVM);
3644	PGM_LOCK_VOID(pVM);
3645	}
3646	}
3647	}
3648	PGM_UNLOCK(pVM);
3649
3650	RT_BZERO(paLocks, sizeof(paLocks[0]) * cPages);
3651	}
3652	#endif /* IN_RING3 */
3653
3654
3655	/**
3656	* Release the internal mapping of a guest page.
3657	*
3658	* This is the counter part of pgmPhysGCPhys2CCPtrInternalEx and
3659	* pgmPhysGCPhys2CCPtrInternalReadOnly.
3660	*
3661	* @param pVM The cross context VM structure.
3662	* @param pLock The lock structure initialized by the mapping function.
3663	*
3664	* @remarks Caller must hold the PGM lock.
3665	*/
3666	void pgmPhysReleaseInternalPageMappingLock(PVMCC pVM, PPGMPAGEMAPLOCK pLock)
3667	{
3668	PGM_LOCK_ASSERT_OWNER(pVM);
3669	PGMPhysReleasePageMappingLock(pVM, pLock); /* lazy for now */
3670	}
3671
3672
3673	/**
3674	* Converts a GC physical address to a HC ring-3 pointer.
3675	*
3676	* @returns VINF_SUCCESS on success.
3677	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
3678	* page but has no physical backing.
3679	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
3680	* GC physical address.
3681	* @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
3682	* a dynamic ram chunk boundary
3683	*
3684	* @param pVM The cross context VM structure.
3685	* @param GCPhys The GC physical address to convert.
3686	* @param pR3Ptr Where to store the R3 pointer on success.
3687	*
3688	* @deprecated Avoid when possible!
3689	*/
3690	int pgmPhysGCPhys2R3Ptr(PVMCC pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
3691	{
3692	/** @todo this is kind of hacky and needs some more work. */
3693	#ifndef DEBUG_sandervl
3694	VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
3695	#endif
3696
3697	Log(("pgmPhysGCPhys2R3Ptr(,%RGp,): dont use this API!\n", GCPhys)); /** @todo eliminate this API! */
3698	PGM_LOCK_VOID(pVM);
3699
3700	PPGMRAMRANGE pRam;
3701	PPGMPAGE pPage;
3702	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
3703	if (RT_SUCCESS(rc))
3704	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr);
3705
3706	PGM_UNLOCK(pVM);
3707	Assert(rc <= VINF_SUCCESS);
3708	return rc;
3709	}
3710
3711
3712	/**
3713	* Special lockless guest physical to current context pointer convertor.
3714	*
3715	* This is mainly for the page table walking and such.
3716	*/
3717	int pgmPhysGCPhys2CCPtrLockless(PVMCPUCC pVCpu, RTGCPHYS GCPhys, void **ppv)
3718	{
3719	VMCPU_ASSERT_EMT(pVCpu);
3720
3721	/*
3722	* Get the RAM range and page structure.
3723	*/
3724	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3725	PGMRAMRANGE volatile *pRam;
3726	PGMPAGE volatile *pPage;
3727	int rc = pgmPhysGetPageAndRangeExLockless(pVM, pVCpu, GCPhys, &pPage, &pRam);
3728	if (RT_SUCCESS(rc))
3729	{
3730	/*
3731	* Now, make sure it's writable (typically it is).
3732	*/
3733	if (RT_LIKELY(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED))
3734	{ /* likely, typically */ }
3735	else
3736	{
3737	PGM_LOCK_VOID(pVM);
3738	rc = pgmPhysPageMakeWritable(pVM, (PPGMPAGE)pPage, GCPhys);
3739	if (RT_SUCCESS(rc))
3740	rc = pgmPhysGetPageAndRangeExLockless(pVM, pVCpu, GCPhys, &pPage, &pRam);
3741	PGM_UNLOCK(pVM);
3742	if (RT_FAILURE(rc))
3743	return rc;
3744	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
3745	}
3746	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0 \|\| PGM_IS_IN_NEM_MODE(pVM));
3747
3748	/*
3749	* Get the mapping address.
3750	*/
3751	uint8_t *pb;
3752	#ifdef IN_RING3
3753	if (PGM_IS_IN_NEM_MODE(pVM))
3754	pb = &pRam->pbR3[(RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << GUEST_PAGE_SHIFT];
3755	else
3756	#endif
3757	{
3758	#ifdef IN_RING3
3759	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
3760	PPGMPAGEMAPTLBE pTlbe;
3761	rc = pgmPhysPageQueryLocklessTlbeWithPage(pVCpu, (PPGMPAGE)pPage, GCPhys, &pTlbe);
3762	AssertLogRelRCReturn(rc, rc);
3763	pb = (uint8_t *)pTlbe->pv;
3764	RT_NOREF(pVM);
3765	# endif
3766	#else /** @todo a safe lockless page TLB in ring-0 needs the to ensure it gets the right invalidations. later. */
3767	PGM_LOCK(pVM);
3768	PPGMPAGEMAPTLBE pTlbe;
3769	rc = pgmPhysPageQueryTlbeWithPage(pVM, (PPGMPAGE)pPage, GCPhys, &pTlbe);
3770	AssertLogRelRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
3771	pb = (uint8_t *)pTlbe->pv;
3772	PGM_UNLOCK(pVM);
3773	RT_NOREF(pVCpu);
3774	#endif
3775	}
3776	ppv = (void )((uintptr_t)pb \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3777	return VINF_SUCCESS;
3778	}
3779	Assert(rc <= VINF_SUCCESS);
3780	return rc;
3781	}
3782
3783
3784	/**
3785	* Converts a guest pointer to a GC physical address.
3786	*
3787	* This uses the current CR3/CR0/CR4 of the guest.
3788	*
3789	* @returns VBox status code.
3790	* @param pVCpu The cross context virtual CPU structure.
3791	* @param GCPtr The guest pointer to convert.
3792	* @param pGCPhys Where to store the GC physical address.
3793	* @thread EMT(pVCpu)
3794	*/
3795	VMMDECL(int) PGMPhysGCPtr2GCPhys(PVMCPUCC pVCpu, RTGCPTR GCPtr, PRTGCPHYS pGCPhys)
3796	{
3797	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
3798	PGMPTWALK Walk;
3799	int rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtr, &Walk);
3800	if (pGCPhys && RT_SUCCESS(rc))
3801	*pGCPhys = Walk.GCPhys \| ((RTGCUINTPTR)GCPtr & GUEST_PAGE_OFFSET_MASK);
3802	return rc;
3803	}
3804
3805
3806	/**
3807	* Converts a guest pointer to a HC physical address.
3808	*
3809	* This uses the current CR3/CR0/CR4 of the guest.
3810	*
3811	* @returns VBox status code.
3812	* @param pVCpu The cross context virtual CPU structure.
3813	* @param GCPtr The guest pointer to convert.
3814	* @param pHCPhys Where to store the HC physical address.
3815	* @thread EMT(pVCpu)
3816	*/
3817	VMM_INT_DECL(int) PGMPhysGCPtr2HCPhys(PVMCPUCC pVCpu, RTGCPTR GCPtr, PRTHCPHYS pHCPhys)
3818	{
3819	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
3820	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3821	PGMPTWALK Walk;
3822	int rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtr, &Walk);
3823	if (RT_SUCCESS(rc))
3824	rc = PGMPhysGCPhys2HCPhys(pVM, Walk.GCPhys \| ((RTGCUINTPTR)GCPtr & GUEST_PAGE_OFFSET_MASK), pHCPhys);
3825	return rc;
3826	}
3827
3828
3829
3830	#undef LOG_GROUP
3831	#define LOG_GROUP LOG_GROUP_PGM_PHYS_ACCESS
3832
3833
3834	#if defined(IN_RING3) && defined(SOME_UNUSED_FUNCTION)
3835	/**
3836	* Cache PGMPhys memory access
3837	*
3838	* @param pVM The cross context VM structure.
3839	* @param pCache Cache structure pointer
3840	* @param GCPhys GC physical address
3841	* @param pbR3 HC pointer corresponding to physical page
3842	*
3843	* @thread EMT.
3844	*/
3845	static void pgmPhysCacheAdd(PVM pVM, PGMPHYSCACHE pCache, RTGCPHYS GCPhys, uint8_t pbR3)
3846	{
3847	uint32_t iCacheIndex;
3848
3849	Assert(VM_IS_EMT(pVM));
3850
3851	GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
3852	pbR3 = (uint8_t *)((uintptr_t)pbR3 & ~(uintptr_t)GUEST_PAGE_OFFSET_MASK);
3853
3854	iCacheIndex = ((GCPhys >> GUEST_PAGE_SHIFT) & PGM_MAX_PHYSCACHE_ENTRIES_MASK);
3855
3856	ASMBitSet(&pCache->aEntries, iCacheIndex);
3857
3858	pCache->Entry[iCacheIndex].GCPhys = GCPhys;
3859	pCache->Entry[iCacheIndex].pbR3 = pbR3;
3860	}
3861	#endif /* IN_RING3 */
3862
3863
3864	/**
3865	* Deals with reading from a page with one or more ALL access handlers.
3866	*
3867	* @returns Strict VBox status code in ring-0 and raw-mode, ignorable in ring-3.
3868	* See PGM_HANDLER_PHYS_IS_VALID_STATUS and
3869	* PGM_HANDLER_VIRT_IS_VALID_STATUS for details.
3870	*
3871	* @param pVM The cross context VM structure.
3872	* @param pPage The page descriptor.
3873	* @param GCPhys The physical address to start reading at.
3874	* @param pvBuf Where to put the bits we read.
3875	* @param cb How much to read - less or equal to a page.
3876	* @param enmOrigin The origin of this call.
3877	*/
3878	static VBOXSTRICTRC pgmPhysReadHandler(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void *pvBuf, size_t cb,
3879	PGMACCESSORIGIN enmOrigin)
3880	{
3881	/*
3882	* The most frequent access here is MMIO and shadowed ROM.
3883	* The current code ASSUMES all these access handlers covers full pages!
3884	*/
3885
3886	/*
3887	* Whatever we do we need the source page, map it first.
3888	*/
3889	PGMPAGEMAPLOCK PgMpLck;
3890	const void *pvSrc = NULL;
3891	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvSrc, &PgMpLck);
3892	/** @todo Check how this can work for MMIO pages? */
3893	if (RT_FAILURE(rc))
3894	{
3895	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
3896	GCPhys, pPage, rc));
3897	memset(pvBuf, 0xff, cb);
3898	return VINF_SUCCESS;
3899	}
3900
3901	VBOXSTRICTRC rcStrict = VINF_PGM_HANDLER_DO_DEFAULT;
3902
3903	/*
3904	* Deal with any physical handlers.
3905	*/
3906	PVMCPUCC pVCpu = VMMGetCpu(pVM);
3907	if ( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL
3908	\|\| PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
3909	{
3910	PPGMPHYSHANDLER pCur;
3911	rc = pgmHandlerPhysicalLookup(pVM, GCPhys, &pCur);
3912	if (RT_SUCCESS(rc))
3913	{
3914	Assert(pCur && GCPhys >= pCur->Key && GCPhys <= pCur->KeyLast);
3915	Assert((pCur->Key & GUEST_PAGE_OFFSET_MASK) == 0);
3916	Assert((pCur->KeyLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK);
3917	#ifndef IN_RING3
3918	if (enmOrigin != PGMACCESSORIGIN_IEM)
3919	{
3920	/* Cannot reliably handle informational status codes in this context */
3921	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
3922	return VERR_PGM_PHYS_WR_HIT_HANDLER;
3923	}
3924	#endif
3925	PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
3926	PFNPGMPHYSHANDLER const pfnHandler = pCurType->pfnHandler; Assert(pfnHandler);
3927	uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
3928	: (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
3929
3930	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cb, pPage, R3STRING(pCur->pszDesc) ));
3931	STAM_PROFILE_START(&pCur->Stat, h);
3932	PGM_LOCK_ASSERT_OWNER(pVM);
3933
3934	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
3935	PGM_UNLOCK(pVM);
3936	/* If the access origins with a device, make sure the buffer is initialized
3937	as a guard against leaking heap, stack and other info via badly written
3938	MMIO handling. @bugref{10651} */
3939	if (enmOrigin == PGMACCESSORIGIN_DEVICE)
3940	memset(pvBuf, 0xff, cb);
3941	rcStrict = pfnHandler(pVM, pVCpu, GCPhys, (void *)pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, enmOrigin, uUser);
3942	PGM_LOCK_VOID(pVM);
3943
3944	STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
3945	pCur = NULL; /* might not be valid anymore. */
3946	AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict, false),
3947	("rcStrict=%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys));
3948	if ( rcStrict != VINF_PGM_HANDLER_DO_DEFAULT
3949	&& !PGM_PHYS_RW_IS_SUCCESS(rcStrict))
3950	{
3951	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
3952	return rcStrict;
3953	}
3954	}
3955	else if (rc == VERR_NOT_FOUND)
3956	AssertLogRelMsgFailed(("rc=%Rrc GCPhys=%RGp cb=%#x\n", rc, GCPhys, cb));
3957	else
3958	AssertLogRelMsgFailedReturn(("rc=%Rrc GCPhys=%RGp cb=%#x\n", rc, GCPhys, cb), rc);
3959	}
3960
3961	/*
3962	* Take the default action.
3963	*/
3964	if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
3965	{
3966	memcpy(pvBuf, pvSrc, cb);
3967	rcStrict = VINF_SUCCESS;
3968	}
3969	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
3970	return rcStrict;
3971	}
3972
3973
3974	/**
3975	* Read physical memory.
3976	*
3977	* This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
3978	* want to ignore those.
3979	*
3980	* @returns Strict VBox status code in raw-mode and ring-0, normal VBox status
3981	* code in ring-3. Use PGM_PHYS_RW_IS_SUCCESS to check.
3982	* @retval VINF_SUCCESS in all context - read completed.
3983	*
3984	* @retval VINF_EM_OFF in RC and R0 - read completed.
3985	* @retval VINF_EM_SUSPEND in RC and R0 - read completed.
3986	* @retval VINF_EM_RESET in RC and R0 - read completed.
3987	* @retval VINF_EM_HALT in RC and R0 - read completed.
3988	* @retval VINF_SELM_SYNC_GDT in RC only - read completed.
3989	*
3990	* @retval VINF_EM_DBG_STOP in RC and R0 - read completed.
3991	* @retval VINF_EM_DBG_BREAKPOINT in RC and R0 - read completed.
3992	* @retval VINF_EM_RAW_EMULATE_INSTR in RC and R0 only.
3993	*
3994	* @retval VINF_IOM_R3_MMIO_READ in RC and R0.
3995	* @retval VINF_IOM_R3_MMIO_READ_WRITE in RC and R0.
3996	*
3997	* @retval VINF_PATM_CHECK_PATCH_PAGE in RC only.
3998	*
3999	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in RC and R0 for access origins that
4000	* haven't been cleared for strict status codes yet.
4001	*
4002	* @param pVM The cross context VM structure.
4003	* @param GCPhys Physical address start reading from.
4004	* @param pvBuf Where to put the read bits.
4005	* @param cbRead How many bytes to read.
4006	* @param enmOrigin The origin of this call.
4007	*/
4008	VMMDECL(VBOXSTRICTRC) PGMPhysRead(PVMCC pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead, PGMACCESSORIGIN enmOrigin)
4009	{
4010	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
4011	LogFlow(("PGMPhysRead: %RGp %d\n", GCPhys, cbRead));
4012
4013	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysRead));
4014	STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysReadBytes), cbRead);
4015
4016	PGM_LOCK_VOID(pVM);
4017
4018	/*
4019	* Copy loop on ram ranges.
4020	*/
4021	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
4022	for (;;)
4023	{
4024	PPGMRAMRANGE const pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
4025
4026	/* Inside range or not? */
4027	if (pRam && GCPhys >= pRam->GCPhys)
4028	{
4029	/*
4030	* Must work our way thru this page by page.
4031	*/
4032	RTGCPHYS off = GCPhys - pRam->GCPhys;
4033	while (off < pRam->cb)
4034	{
4035	unsigned iPage = off >> GUEST_PAGE_SHIFT;
4036	PPGMPAGE pPage = &pRam->aPages[iPage];
4037	size_t cb = GUEST_PAGE_SIZE - (off & GUEST_PAGE_OFFSET_MASK);
4038	if (cb > cbRead)
4039	cb = cbRead;
4040
4041	/*
4042	* Normal page? Get the pointer to it.
4043	*/
4044	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
4045	&& !PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
4046	{
4047	/*
4048	* Get the pointer to the page.
4049	*/
4050	PGMPAGEMAPLOCK PgMpLck;
4051	const void *pvSrc;
4052	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
4053	if (RT_SUCCESS(rc))
4054	{
4055	memcpy(pvBuf, pvSrc, cb);
4056	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
4057	}
4058	else
4059	{
4060	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
4061	pRam->GCPhys + off, pPage, rc));
4062	memset(pvBuf, 0xff, cb);
4063	}
4064	}
4065	/*
4066	* Have ALL/MMIO access handlers.
4067	*/
4068	else
4069	{
4070	VBOXSTRICTRC rcStrict2 = pgmPhysReadHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb, enmOrigin);
4071	if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
4072	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
4073	else
4074	{
4075	/* Set the remaining buffer to a known value. */
4076	memset(pvBuf, 0xff, cbRead);
4077	PGM_UNLOCK(pVM);
4078	return rcStrict2;
4079	}
4080	}
4081
4082	/* next page */
4083	if (cb >= cbRead)
4084	{
4085	PGM_UNLOCK(pVM);
4086	return rcStrict;
4087	}
4088	cbRead -= cb;
4089	off += cb;
4090	pvBuf = (char *)pvBuf + cb;
4091	} /* walk pages in ram range. */
4092
4093	GCPhys = pRam->GCPhysLast + 1;
4094	}
4095	else
4096	{
4097	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
4098
4099	/*
4100	* Unassigned address space.
4101	*/
4102	size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
4103	if (cb >= cbRead)
4104	{
4105	memset(pvBuf, 0xff, cbRead);
4106	break;
4107	}
4108	memset(pvBuf, 0xff, cb);
4109
4110	cbRead -= cb;
4111	pvBuf = (char *)pvBuf + cb;
4112	GCPhys += cb;
4113	}
4114
4115	} /* Ram range walk */
4116
4117	PGM_UNLOCK(pVM);
4118	return rcStrict;
4119	}
4120
4121
4122	/**
4123	* Deals with writing to a page with one or more WRITE or ALL access handlers.
4124	*
4125	* @returns Strict VBox status code in ring-0 and raw-mode, ignorable in ring-3.
4126	* See PGM_HANDLER_PHYS_IS_VALID_STATUS and
4127	* PGM_HANDLER_VIRT_IS_VALID_STATUS for details.
4128	*
4129	* @param pVM The cross context VM structure.
4130	* @param pPage The page descriptor.
4131	* @param GCPhys The physical address to start writing at.
4132	* @param pvBuf What to write.
4133	* @param cbWrite How much to write - less or equal to a page.
4134	* @param enmOrigin The origin of this call.
4135	*/
4136	static VBOXSTRICTRC pgmPhysWriteHandler(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const *pvBuf, size_t cbWrite,
4137	PGMACCESSORIGIN enmOrigin)
4138	{
4139	PGMPAGEMAPLOCK PgMpLck;
4140	void *pvDst = NULL;
4141	VBOXSTRICTRC rcStrict;
4142
4143	/*
4144	* Give priority to physical handlers (like #PF does).
4145	*
4146	* Hope for a lonely physical handler first that covers the whole write
4147	* area. This should be a pretty frequent case with MMIO and the heavy
4148	* usage of full page handlers in the page pool.
4149	*/
4150	PVMCPUCC pVCpu = VMMGetCpu(pVM);
4151	PPGMPHYSHANDLER pCur;
4152	rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhys, &pCur);
4153	if (RT_SUCCESS(rcStrict))
4154	{
4155	Assert(GCPhys >= pCur->Key && GCPhys <= pCur->KeyLast);
4156	#ifndef IN_RING3
4157	if (enmOrigin != PGMACCESSORIGIN_IEM)
4158	/* Cannot reliably handle informational status codes in this context */
4159	return VERR_PGM_PHYS_WR_HIT_HANDLER;
4160	#endif
4161	size_t cbRange = pCur->KeyLast - GCPhys + 1;
4162	if (cbRange > cbWrite)
4163	cbRange = cbWrite;
4164
4165	Assert(PGMPHYSHANDLER_GET_TYPE(pVM, pCur)->pfnHandler);
4166	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n",
4167	GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
4168	if (!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
4169	rcStrict = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
4170	else
4171	rcStrict = VINF_SUCCESS;
4172	if (RT_SUCCESS(rcStrict))
4173	{
4174	PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
4175	PFNPGMPHYSHANDLER const pfnHandler = pCurType->pfnHandler;
4176	uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
4177	: (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
4178	STAM_PROFILE_START(&pCur->Stat, h);
4179
4180	/* Most handlers will want to release the PGM lock for deadlock prevention
4181	(esp. MMIO), though some PGM internal ones like the page pool and MMIO2
4182	dirty page trackers will want to keep it for performance reasons. */
4183	PGM_LOCK_ASSERT_OWNER(pVM);
4184	if (pCurType->fKeepPgmLock)
4185	rcStrict = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser);
4186	else
4187	{
4188	PGM_UNLOCK(pVM);
4189	rcStrict = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser);
4190	PGM_LOCK_VOID(pVM);
4191	}
4192
4193	STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
4194	pCur = NULL; /* might not be valid anymore. */
4195	if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
4196	{
4197	if (pvDst)
4198	memcpy(pvDst, pvBuf, cbRange);
4199	rcStrict = VINF_SUCCESS;
4200	}
4201	else
4202	AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict, true),
4203	("rcStrict=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n",
4204	VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pCur ? R3STRING(pCur->pszDesc) : ""));
4205	}
4206	else
4207	AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
4208	GCPhys, pPage, VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
4209	if (RT_LIKELY(cbRange == cbWrite) \|\| !PGM_PHYS_RW_IS_SUCCESS(rcStrict))
4210	{
4211	if (pvDst)
4212	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
4213	return rcStrict;
4214	}
4215
4216	/* more fun to be had below */
4217	cbWrite -= cbRange;
4218	GCPhys += cbRange;
4219	pvBuf = (uint8_t *)pvBuf + cbRange;
4220	pvDst = (uint8_t *)pvDst + cbRange;
4221	}
4222	else if (rcStrict == VERR_NOT_FOUND) /* The handler is somewhere else in the page, deal with it below. */
4223	rcStrict = VINF_SUCCESS;
4224	else
4225	AssertMsgFailedReturn(("rcStrict=%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys), rcStrict);
4226	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)); /* MMIO handlers are all GUEST_PAGE_SIZEed! */
4227
4228	/*
4229	* Deal with all the odd ends (used to be deal with virt+phys).
4230	*/
4231	Assert(rcStrict != VINF_PGM_HANDLER_DO_DEFAULT);
4232
4233	/* We need a writable destination page. */
4234	if (!pvDst)
4235	{
4236	int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
4237	AssertLogRelMsgReturn(RT_SUCCESS(rc2),
4238	("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n", GCPhys, pPage, rc2),
4239	rc2);
4240	}
4241
4242	/** @todo clean up this code some more now there are no virtual handlers any
4243	* more. */
4244	/* The loop state (big + ugly). */
4245	PPGMPHYSHANDLER pPhys = NULL;
4246	uint32_t offPhys = GUEST_PAGE_SIZE;
4247	uint32_t offPhysLast = GUEST_PAGE_SIZE;
4248	bool fMorePhys = PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage);
4249
4250	/* The loop. */
4251	for (;;)
4252	{
4253	if (fMorePhys && !pPhys)
4254	{
4255	rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhys, &pPhys);
4256	if (RT_SUCCESS_NP(rcStrict))
4257	{
4258	offPhys = 0;
4259	offPhysLast = pPhys->KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
4260	}
4261	else
4262	{
4263	AssertMsgReturn(rcStrict == VERR_NOT_FOUND, ("%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys), rcStrict);
4264
4265	rcStrict = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookupMatchingOrAbove(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator,
4266	GCPhys, &pPhys);
4267	AssertMsgReturn(RT_SUCCESS(rcStrict) \|\| rcStrict == VERR_NOT_FOUND,
4268	("%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys), rcStrict);
4269
4270	if ( RT_SUCCESS(rcStrict)
4271	&& pPhys->Key <= GCPhys + (cbWrite - 1))
4272	{
4273	offPhys = pPhys->Key - GCPhys;
4274	offPhysLast = pPhys->KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
4275	Assert(pPhys->KeyLast - pPhys->Key < _4G);
4276	}
4277	else
4278	{
4279	pPhys = NULL;
4280	fMorePhys = false;
4281	offPhys = offPhysLast = GUEST_PAGE_SIZE;
4282	}
4283	}
4284	}
4285
4286	/*
4287	* Handle access to space without handlers (that's easy).
4288	*/
4289	VBOXSTRICTRC rcStrict2 = VINF_PGM_HANDLER_DO_DEFAULT;
4290	uint32_t cbRange = (uint32_t)cbWrite;
4291	Assert(cbRange == cbWrite);
4292
4293	/*
4294	* Physical handler.
4295	*/
4296	if (!offPhys)
4297	{
4298	#ifndef IN_RING3
4299	if (enmOrigin != PGMACCESSORIGIN_IEM)
4300	/* Cannot reliably handle informational status codes in this context */
4301	return VERR_PGM_PHYS_WR_HIT_HANDLER;
4302	#endif
4303	if (cbRange > offPhysLast + 1)
4304	cbRange = offPhysLast + 1;
4305
4306	PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pPhys);
4307	PFNPGMPHYSHANDLER const pfnHandler = pCurType->pfnHandler;
4308	uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pPhys->uUser
4309	: (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pPhys->uUser);
4310
4311	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc) ));
4312	STAM_PROFILE_START(&pPhys->Stat, h);
4313
4314	/* Most handlers will want to release the PGM lock for deadlock prevention
4315	(esp. MMIO), though some PGM internal ones like the page pool and MMIO2
4316	dirty page trackers will want to keep it for performance reasons. */
4317	PGM_LOCK_ASSERT_OWNER(pVM);
4318	if (pCurType->fKeepPgmLock)
4319	rcStrict2 = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser);
4320	else
4321	{
4322	PGM_UNLOCK(pVM);
4323	rcStrict2 = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser);
4324	PGM_LOCK_VOID(pVM);
4325	}
4326
4327	STAM_PROFILE_STOP(&pPhys->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
4328	pPhys = NULL; /* might not be valid anymore. */
4329	AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict2, true),
4330	("rcStrict2=%Rrc (rcStrict=%Rrc) GCPhys=%RGp pPage=%R[pgmpage] %s\n", VBOXSTRICTRC_VAL(rcStrict2),
4331	VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pPhys ? R3STRING(pPhys->pszDesc) : ""));
4332	}
4333
4334	/*
4335	* Execute the default action and merge the status codes.
4336	*/
4337	if (rcStrict2 == VINF_PGM_HANDLER_DO_DEFAULT)
4338	{
4339	memcpy(pvDst, pvBuf, cbRange);
4340	rcStrict2 = VINF_SUCCESS;
4341	}
4342	else if (!PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
4343	{
4344	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
4345	return rcStrict2;
4346	}
4347	else
4348	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
4349
4350	/*
4351	* Advance if we've got more stuff to do.
4352	*/
4353	if (cbRange >= cbWrite)
4354	{
4355	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
4356	return rcStrict;
4357	}
4358
4359
4360	cbWrite -= cbRange;
4361	GCPhys += cbRange;
4362	pvBuf = (uint8_t *)pvBuf + cbRange;
4363	pvDst = (uint8_t *)pvDst + cbRange;
4364
4365	offPhys -= cbRange;
4366	offPhysLast -= cbRange;
4367	}
4368	}
4369
4370
4371	/**
4372	* Write to physical memory.
4373	*
4374	* This API respects access handlers and MMIO. Use PGMPhysSimpleWriteGCPhys() if you
4375	* want to ignore those.
4376	*
4377	* @returns Strict VBox status code in raw-mode and ring-0, normal VBox status
4378	* code in ring-3. Use PGM_PHYS_RW_IS_SUCCESS to check.
4379	* @retval VINF_SUCCESS in all context - write completed.
4380	*
4381	* @retval VINF_EM_OFF in RC and R0 - write completed.
4382	* @retval VINF_EM_SUSPEND in RC and R0 - write completed.
4383	* @retval VINF_EM_RESET in RC and R0 - write completed.
4384	* @retval VINF_EM_HALT in RC and R0 - write completed.
4385	* @retval VINF_SELM_SYNC_GDT in RC only - write completed.
4386	*
4387	* @retval VINF_EM_DBG_STOP in RC and R0 - write completed.
4388	* @retval VINF_EM_DBG_BREAKPOINT in RC and R0 - write completed.
4389	* @retval VINF_EM_RAW_EMULATE_INSTR in RC and R0 only.
4390	*
4391	* @retval VINF_IOM_R3_MMIO_WRITE in RC and R0.
4392	* @retval VINF_IOM_R3_MMIO_READ_WRITE in RC and R0.
4393	* @retval VINF_IOM_R3_MMIO_COMMIT_WRITE in RC and R0.
4394	*
4395	* @retval VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT in RC only - write completed.
4396	* @retval VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT in RC only.
4397	* @retval VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT in RC only.
4398	* @retval VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT in RC only.
4399	* @retval VINF_CSAM_PENDING_ACTION in RC only.
4400	* @retval VINF_PATM_CHECK_PATCH_PAGE in RC only.
4401	*
4402	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in RC and R0 for access origins that
4403	* haven't been cleared for strict status codes yet.
4404	*
4405	*
4406	* @param pVM The cross context VM structure.
4407	* @param GCPhys Physical address to write to.
4408	* @param pvBuf What to write.
4409	* @param cbWrite How many bytes to write.
4410	* @param enmOrigin Who is calling.
4411	*/
4412	VMMDECL(VBOXSTRICTRC) PGMPhysWrite(PVMCC pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, PGMACCESSORIGIN enmOrigin)
4413	{
4414	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites, ("Calling PGMPhysWrite after pgmR3Save()! enmOrigin=%d\n", enmOrigin));
4415	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
4416	LogFlow(("PGMPhysWrite: %RGp %d\n", GCPhys, cbWrite));
4417
4418	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysWrite));
4419	STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysWriteBytes), cbWrite);
4420
4421	PGM_LOCK_VOID(pVM);
4422
4423	/*
4424	* Copy loop on ram ranges.
4425	*/
4426	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
4427	for (;;)
4428	{
4429	PPGMRAMRANGE const pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
4430
4431	/* Inside range or not? */
4432	if (pRam && GCPhys >= pRam->GCPhys)
4433	{
4434	/*
4435	* Must work our way thru this page by page.
4436	*/
4437	RTGCPTR off = GCPhys - pRam->GCPhys;
4438	while (off < pRam->cb)
4439	{
4440	RTGCPTR iPage = off >> GUEST_PAGE_SHIFT;
4441	PPGMPAGE pPage = &pRam->aPages[iPage];
4442	size_t cb = GUEST_PAGE_SIZE - (off & GUEST_PAGE_OFFSET_MASK);
4443	if (cb > cbWrite)
4444	cb = cbWrite;
4445
4446	/*
4447	* Normal page? Get the pointer to it.
4448	*/
4449	if ( !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
4450	&& !PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
4451	{
4452	PGMPAGEMAPLOCK PgMpLck;
4453	void *pvDst;
4454	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
4455	if (RT_SUCCESS(rc))
4456	{
4457	Assert(!PGM_PAGE_IS_BALLOONED(pPage));
4458	memcpy(pvDst, pvBuf, cb);
4459	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
4460	}
4461	/* Ignore writes to ballooned pages. */
4462	else if (!PGM_PAGE_IS_BALLOONED(pPage))
4463	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
4464	pRam->GCPhys + off, pPage, rc));
4465	}
4466	/*
4467	* Active WRITE or ALL access handlers.
4468	*/
4469	else
4470	{
4471	VBOXSTRICTRC rcStrict2 = pgmPhysWriteHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb, enmOrigin);
4472	if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
4473	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
4474	else
4475	{
4476	PGM_UNLOCK(pVM);
4477	return rcStrict2;
4478	}
4479	}
4480
4481	/* next page */
4482	if (cb >= cbWrite)
4483	{
4484	PGM_UNLOCK(pVM);
4485	return rcStrict;
4486	}
4487
4488	cbWrite -= cb;
4489	off += cb;
4490	pvBuf = (const char *)pvBuf + cb;
4491	} /* walk pages in ram range */
4492
4493	GCPhys = pRam->GCPhysLast + 1;
4494	}
4495	else
4496	{
4497	/*
4498	* Unassigned address space, skip it.
4499	*/
4500	if (!pRam)
4501	break;
4502	size_t cb = pRam->GCPhys - GCPhys;
4503	if (cb >= cbWrite)
4504	break;
4505	cbWrite -= cb;
4506	pvBuf = (const char *)pvBuf + cb;
4507	GCPhys += cb;
4508	}
4509
4510	} /* Ram range walk */
4511
4512	PGM_UNLOCK(pVM);
4513	return rcStrict;
4514	}
4515
4516
4517	/**
4518	* Read from guest physical memory by GC physical address, bypassing
4519	* MMIO and access handlers.
4520	*
4521	* @returns VBox status code.
4522	* @param pVM The cross context VM structure.
4523	* @param pvDst The destination address.
4524	* @param GCPhysSrc The source address (GC physical address).
4525	* @param cb The number of bytes to read.
4526	*/
4527	VMMDECL(int) PGMPhysSimpleReadGCPhys(PVMCC pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb)
4528	{
4529	/*
4530	* Treat the first page as a special case.
4531	*/
4532	if (!cb)
4533	return VINF_SUCCESS;
4534
4535	/* map the 1st page */
4536	void const *pvSrc;
4537	PGMPAGEMAPLOCK Lock;
4538	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
4539	if (RT_FAILURE(rc))
4540	return rc;
4541
4542	/* optimize for the case where access is completely within the first page. */
4543	size_t cbPage = GUEST_PAGE_SIZE - (GCPhysSrc & GUEST_PAGE_OFFSET_MASK);
4544	if (RT_LIKELY(cb <= cbPage))
4545	{
4546	memcpy(pvDst, pvSrc, cb);
4547	PGMPhysReleasePageMappingLock(pVM, &Lock);
4548	return VINF_SUCCESS;
4549	}
4550
4551	/* copy to the end of the page. */
4552	memcpy(pvDst, pvSrc, cbPage);
4553	PGMPhysReleasePageMappingLock(pVM, &Lock);
4554	GCPhysSrc += cbPage;
4555	pvDst = (uint8_t *)pvDst + cbPage;
4556	cb -= cbPage;
4557
4558	/*
4559	* Page by page.
4560	*/
4561	for (;;)
4562	{
4563	/* map the page */
4564	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
4565	if (RT_FAILURE(rc))
4566	return rc;
4567
4568	/* last page? */
4569	if (cb <= GUEST_PAGE_SIZE)
4570	{
4571	memcpy(pvDst, pvSrc, cb);
4572	PGMPhysReleasePageMappingLock(pVM, &Lock);
4573	return VINF_SUCCESS;
4574	}
4575
4576	/* copy the entire page and advance */
4577	memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
4578	PGMPhysReleasePageMappingLock(pVM, &Lock);
4579	GCPhysSrc += GUEST_PAGE_SIZE;
4580	pvDst = (uint8_t *)pvDst + GUEST_PAGE_SIZE;
4581	cb -= GUEST_PAGE_SIZE;
4582	}
4583	/* won't ever get here. */
4584	}
4585
4586
4587	/**
4588	* Write to guest physical memory referenced by GC pointer.
4589	* Write memory to GC physical address in guest physical memory.
4590	*
4591	* This will bypass MMIO and access handlers.
4592	*
4593	* @returns VBox status code.
4594	* @param pVM The cross context VM structure.
4595	* @param GCPhysDst The GC physical address of the destination.
4596	* @param pvSrc The source buffer.
4597	* @param cb The number of bytes to write.
4598	*/
4599	VMMDECL(int) PGMPhysSimpleWriteGCPhys(PVMCC pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb)
4600	{
4601	LogFlow(("PGMPhysSimpleWriteGCPhys: %RGp %zu\n", GCPhysDst, cb));
4602
4603	/*
4604	* Treat the first page as a special case.
4605	*/
4606	if (!cb)
4607	return VINF_SUCCESS;
4608
4609	/* map the 1st page */
4610	void *pvDst;
4611	PGMPAGEMAPLOCK Lock;
4612	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
4613	if (RT_FAILURE(rc))
4614	return rc;
4615
4616	/* optimize for the case where access is completely within the first page. */
4617	size_t cbPage = GUEST_PAGE_SIZE - (GCPhysDst & GUEST_PAGE_OFFSET_MASK);
4618	if (RT_LIKELY(cb <= cbPage))
4619	{
4620	memcpy(pvDst, pvSrc, cb);
4621	PGMPhysReleasePageMappingLock(pVM, &Lock);
4622	return VINF_SUCCESS;
4623	}
4624
4625	/* copy to the end of the page. */
4626	memcpy(pvDst, pvSrc, cbPage);
4627	PGMPhysReleasePageMappingLock(pVM, &Lock);
4628	GCPhysDst += cbPage;
4629	pvSrc = (const uint8_t *)pvSrc + cbPage;
4630	cb -= cbPage;
4631
4632	/*
4633	* Page by page.
4634	*/
4635	for (;;)
4636	{
4637	/* map the page */
4638	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
4639	if (RT_FAILURE(rc))
4640	return rc;
4641
4642	/* last page? */
4643	if (cb <= GUEST_PAGE_SIZE)
4644	{
4645	memcpy(pvDst, pvSrc, cb);
4646	PGMPhysReleasePageMappingLock(pVM, &Lock);
4647	return VINF_SUCCESS;
4648	}
4649
4650	/* copy the entire page and advance */
4651	memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
4652	PGMPhysReleasePageMappingLock(pVM, &Lock);
4653	GCPhysDst += GUEST_PAGE_SIZE;
4654	pvSrc = (const uint8_t *)pvSrc + GUEST_PAGE_SIZE;
4655	cb -= GUEST_PAGE_SIZE;
4656	}
4657	/* won't ever get here. */
4658	}
4659
4660
4661	/**
4662	* Read from guest physical memory referenced by GC pointer.
4663	*
4664	* This function uses the current CR3/CR0/CR4 of the guest and will
4665	* bypass access handlers and not set any accessed bits.
4666	*
4667	* @returns VBox status code.
4668	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4669	* @param pvDst The destination address.
4670	* @param GCPtrSrc The source address (GC pointer).
4671	* @param cb The number of bytes to read.
4672	*/
4673	VMMDECL(int) PGMPhysSimpleReadGCPtr(PVMCPUCC pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
4674	{
4675	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4676	/** @todo fix the macro / state handling: VMCPU_ASSERT_EMT_OR_GURU(pVCpu); */
4677
4678	/*
4679	* Treat the first page as a special case.
4680	*/
4681	if (!cb)
4682	return VINF_SUCCESS;
4683
4684	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleRead));
4685	STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleReadBytes), cb);
4686
4687	/* Take the PGM lock here, because many called functions take the lock for a very short period. That's counter-productive
4688	* when many VCPUs are fighting for the lock.
4689	*/
4690	PGM_LOCK_VOID(pVM);
4691
4692	/* map the 1st page */
4693	void const *pvSrc;
4694	PGMPAGEMAPLOCK Lock;
4695	int rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
4696	if (RT_FAILURE(rc))
4697	{
4698	PGM_UNLOCK(pVM);
4699	return rc;
4700	}
4701
4702	/* optimize for the case where access is completely within the first page. */
4703	size_t cbPage = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
4704	if (RT_LIKELY(cb <= cbPage))
4705	{
4706	memcpy(pvDst, pvSrc, cb);
4707	PGMPhysReleasePageMappingLock(pVM, &Lock);
4708	PGM_UNLOCK(pVM);
4709	return VINF_SUCCESS;
4710	}
4711
4712	/* copy to the end of the page. */
4713	memcpy(pvDst, pvSrc, cbPage);
4714	PGMPhysReleasePageMappingLock(pVM, &Lock);
4715	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + cbPage);
4716	pvDst = (uint8_t *)pvDst + cbPage;
4717	cb -= cbPage;
4718
4719	/*
4720	* Page by page.
4721	*/
4722	for (;;)
4723	{
4724	/* map the page */
4725	rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
4726	if (RT_FAILURE(rc))
4727	{
4728	PGM_UNLOCK(pVM);
4729	return rc;
4730	}
4731
4732	/* last page? */
4733	if (cb <= GUEST_PAGE_SIZE)
4734	{
4735	memcpy(pvDst, pvSrc, cb);
4736	PGMPhysReleasePageMappingLock(pVM, &Lock);
4737	PGM_UNLOCK(pVM);
4738	return VINF_SUCCESS;
4739	}
4740
4741	/* copy the entire page and advance */
4742	memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
4743	PGMPhysReleasePageMappingLock(pVM, &Lock);
4744	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + GUEST_PAGE_SIZE);
4745	pvDst = (uint8_t *)pvDst + GUEST_PAGE_SIZE;
4746	cb -= GUEST_PAGE_SIZE;
4747	}
4748	/* won't ever get here. */
4749	}
4750
4751
4752	/**
4753	* Write to guest physical memory referenced by GC pointer.
4754	*
4755	* This function uses the current CR3/CR0/CR4 of the guest and will
4756	* bypass access handlers and not set dirty or accessed bits.
4757	*
4758	* @returns VBox status code.
4759	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4760	* @param GCPtrDst The destination address (GC pointer).
4761	* @param pvSrc The source address.
4762	* @param cb The number of bytes to write.
4763	*/
4764	VMMDECL(int) PGMPhysSimpleWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
4765	{
4766	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4767	VMCPU_ASSERT_EMT(pVCpu);
4768
4769	/*
4770	* Treat the first page as a special case.
4771	*/
4772	if (!cb)
4773	return VINF_SUCCESS;
4774
4775	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleWrite));
4776	STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleWriteBytes), cb);
4777
4778	/* map the 1st page */
4779	void *pvDst;
4780	PGMPAGEMAPLOCK Lock;
4781	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
4782	if (RT_FAILURE(rc))
4783	return rc;
4784
4785	/* optimize for the case where access is completely within the first page. */
4786	size_t cbPage = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
4787	if (RT_LIKELY(cb <= cbPage))
4788	{
4789	memcpy(pvDst, pvSrc, cb);
4790	PGMPhysReleasePageMappingLock(pVM, &Lock);
4791	return VINF_SUCCESS;
4792	}
4793
4794	/* copy to the end of the page. */
4795	memcpy(pvDst, pvSrc, cbPage);
4796	PGMPhysReleasePageMappingLock(pVM, &Lock);
4797	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
4798	pvSrc = (const uint8_t *)pvSrc + cbPage;
4799	cb -= cbPage;
4800
4801	/*
4802	* Page by page.
4803	*/
4804	for (;;)
4805	{
4806	/* map the page */
4807	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
4808	if (RT_FAILURE(rc))
4809	return rc;
4810
4811	/* last page? */
4812	if (cb <= GUEST_PAGE_SIZE)
4813	{
4814	memcpy(pvDst, pvSrc, cb);
4815	PGMPhysReleasePageMappingLock(pVM, &Lock);
4816	return VINF_SUCCESS;
4817	}
4818
4819	/* copy the entire page and advance */
4820	memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
4821	PGMPhysReleasePageMappingLock(pVM, &Lock);
4822	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + GUEST_PAGE_SIZE);
4823	pvSrc = (const uint8_t *)pvSrc + GUEST_PAGE_SIZE;
4824	cb -= GUEST_PAGE_SIZE;
4825	}
4826	/* won't ever get here. */
4827	}
4828
4829
4830	/**
4831	* Write to guest physical memory referenced by GC pointer and update the PTE.
4832	*
4833	* This function uses the current CR3/CR0/CR4 of the guest and will
4834	* bypass access handlers but will set any dirty and accessed bits in the PTE.
4835	*
4836	* If you don't want to set the dirty bit, use PGMPhysSimpleWriteGCPtr().
4837	*
4838	* @returns VBox status code.
4839	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4840	* @param GCPtrDst The destination address (GC pointer).
4841	* @param pvSrc The source address.
4842	* @param cb The number of bytes to write.
4843	*/
4844	VMMDECL(int) PGMPhysSimpleDirtyWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
4845	{
4846	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4847	VMCPU_ASSERT_EMT(pVCpu);
4848
4849	/*
4850	* Treat the first page as a special case.
4851	* Btw. this is the same code as in PGMPhyssimpleWriteGCPtr excep for the PGMGstModifyPage.
4852	*/
4853	if (!cb)
4854	return VINF_SUCCESS;
4855
4856	/* map the 1st page */
4857	void *pvDst;
4858	PGMPAGEMAPLOCK Lock;
4859	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
4860	if (RT_FAILURE(rc))
4861	return rc;
4862
4863	/* optimize for the case where access is completely within the first page. */
4864	size_t cbPage = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
4865	if (RT_LIKELY(cb <= cbPage))
4866	{
4867	memcpy(pvDst, pvSrc, cb);
4868	PGMPhysReleasePageMappingLock(pVM, &Lock);
4869	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
4870	return VINF_SUCCESS;
4871	}
4872
4873	/* copy to the end of the page. */
4874	memcpy(pvDst, pvSrc, cbPage);
4875	PGMPhysReleasePageMappingLock(pVM, &Lock);
4876	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
4877	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
4878	pvSrc = (const uint8_t *)pvSrc + cbPage;
4879	cb -= cbPage;
4880
4881	/*
4882	* Page by page.
4883	*/
4884	for (;;)
4885	{
4886	/* map the page */
4887	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
4888	if (RT_FAILURE(rc))
4889	return rc;
4890
4891	/* last page? */
4892	if (cb <= GUEST_PAGE_SIZE)
4893	{
4894	memcpy(pvDst, pvSrc, cb);
4895	PGMPhysReleasePageMappingLock(pVM, &Lock);
4896	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
4897	return VINF_SUCCESS;
4898	}
4899
4900	/* copy the entire page and advance */
4901	memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
4902	PGMPhysReleasePageMappingLock(pVM, &Lock);
4903	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
4904	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + GUEST_PAGE_SIZE);
4905	pvSrc = (const uint8_t *)pvSrc + GUEST_PAGE_SIZE;
4906	cb -= GUEST_PAGE_SIZE;
4907	}
4908	/* won't ever get here. */
4909	}
4910
4911
4912	/**
4913	* Read from guest physical memory referenced by GC pointer.
4914	*
4915	* This function uses the current CR3/CR0/CR4 of the guest and will
4916	* respect access handlers and set accessed bits.
4917	*
4918	* @returns Strict VBox status, see PGMPhysRead for details.
4919	* @retval VERR_PAGE_TABLE_NOT_PRESENT if there is no page mapped at the
4920	* specified virtual address.
4921	*
4922	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4923	* @param pvDst The destination address.
4924	* @param GCPtrSrc The source address (GC pointer).
4925	* @param cb The number of bytes to read.
4926	* @param enmOrigin Who is calling.
4927	* @thread EMT(pVCpu)
4928	*/
4929	VMMDECL(VBOXSTRICTRC) PGMPhysReadGCPtr(PVMCPUCC pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb, PGMACCESSORIGIN enmOrigin)
4930	{
4931	int rc;
4932	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4933	VMCPU_ASSERT_EMT(pVCpu);
4934
4935	/*
4936	* Anything to do?
4937	*/
4938	if (!cb)
4939	return VINF_SUCCESS;
4940
4941	LogFlow(("PGMPhysReadGCPtr: %RGv %zu\n", GCPtrSrc, cb));
4942
4943	/*
4944	* Optimize reads within a single page.
4945	*/
4946	if (((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK) + cb <= GUEST_PAGE_SIZE)
4947	{
4948	/* Convert virtual to physical address + flags */
4949	PGMPTWALK Walk;
4950	rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrSrc, &Walk);
4951	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
4952	RTGCPHYS const GCPhys = Walk.GCPhys \| ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
4953
4954	/* mark the guest page as accessed. */
4955	if (!(Walk.fEffective & X86_PTE_A))
4956	{
4957	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
4958	AssertRC(rc);
4959	}
4960
4961	return PGMPhysRead(pVM, GCPhys, pvDst, cb, enmOrigin);
4962	}
4963
4964	/*
4965	* Page by page.
4966	*/
4967	for (;;)
4968	{
4969	/* Convert virtual to physical address + flags */
4970	PGMPTWALK Walk;
4971	rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrSrc, &Walk);
4972	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
4973	RTGCPHYS const GCPhys = Walk.GCPhys \| ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
4974
4975	/* mark the guest page as accessed. */
4976	if (!(Walk.fEffective & X86_PTE_A))
4977	{
4978	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
4979	AssertRC(rc);
4980	}
4981
4982	/* copy */
4983	size_t cbRead = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
4984	if (cbRead < cb)
4985	{
4986	VBOXSTRICTRC rcStrict = PGMPhysRead(pVM, GCPhys, pvDst, cbRead, enmOrigin);
4987	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
4988	{ /* likely */ }
4989	else
4990	return rcStrict;
4991	}
4992	else /* Last page (cbRead is GUEST_PAGE_SIZE, we only need cb!) */
4993	return PGMPhysRead(pVM, GCPhys, pvDst, cb, enmOrigin);
4994
4995	/* next */
4996	Assert(cb > cbRead);
4997	cb -= cbRead;
4998	pvDst = (uint8_t *)pvDst + cbRead;
4999	GCPtrSrc += cbRead;
5000	}
5001	}
5002
5003
5004	/**
5005	* Write to guest physical memory referenced by GC pointer.
5006	*
5007	* This function uses the current CR3/CR0/CR4 of the guest and will
5008	* respect access handlers and set dirty and accessed bits.
5009	*
5010	* @returns Strict VBox status, see PGMPhysWrite for details.
5011	* @retval VERR_PAGE_TABLE_NOT_PRESENT if there is no page mapped at the
5012	* specified virtual address.
5013	*
5014	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
5015	* @param GCPtrDst The destination address (GC pointer).
5016	* @param pvSrc The source address.
5017	* @param cb The number of bytes to write.
5018	* @param enmOrigin Who is calling.
5019	*/
5020	VMMDECL(VBOXSTRICTRC) PGMPhysWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb, PGMACCESSORIGIN enmOrigin)
5021	{
5022	int rc;
5023	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5024	VMCPU_ASSERT_EMT(pVCpu);
5025
5026	/*
5027	* Anything to do?
5028	*/
5029	if (!cb)
5030	return VINF_SUCCESS;
5031
5032	LogFlow(("PGMPhysWriteGCPtr: %RGv %zu\n", GCPtrDst, cb));
5033
5034	/*
5035	* Optimize writes within a single page.
5036	*/
5037	if (((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK) + cb <= GUEST_PAGE_SIZE)
5038	{
5039	/* Convert virtual to physical address + flags */
5040	PGMPTWALK Walk;
5041	rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrDst, &Walk);
5042	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
5043	RTGCPHYS const GCPhys = Walk.GCPhys \| ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
5044
5045	/* Mention when we ignore X86_PTE_RW... */
5046	if (!(Walk.fEffective & X86_PTE_RW))
5047	Log(("PGMPhysWriteGCPtr: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
5048
5049	/* Mark the guest page as accessed and dirty if necessary. */
5050	if ((Walk.fEffective & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
5051	{
5052	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
5053	AssertRC(rc);
5054	}
5055
5056	return PGMPhysWrite(pVM, GCPhys, pvSrc, cb, enmOrigin);
5057	}
5058
5059	/*
5060	* Page by page.
5061	*/
5062	for (;;)
5063	{
5064	/* Convert virtual to physical address + flags */
5065	PGMPTWALK Walk;
5066	rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrDst, &Walk);
5067	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
5068	RTGCPHYS const GCPhys = Walk.GCPhys \| ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
5069
5070	/* Mention when we ignore X86_PTE_RW... */
5071	if (!(Walk.fEffective & X86_PTE_RW))
5072	Log(("PGMPhysWriteGCPtr: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
5073
5074	/* Mark the guest page as accessed and dirty if necessary. */
5075	if ((Walk.fEffective & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
5076	{
5077	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
5078	AssertRC(rc);
5079	}
5080
5081	/* copy */
5082	size_t cbWrite = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
5083	if (cbWrite < cb)
5084	{
5085	VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM, GCPhys, pvSrc, cbWrite, enmOrigin);
5086	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
5087	{ /* likely */ }
5088	else
5089	return rcStrict;
5090	}
5091	else /* Last page (cbWrite is GUEST_PAGE_SIZE, we only need cb!) */
5092	return PGMPhysWrite(pVM, GCPhys, pvSrc, cb, enmOrigin);
5093
5094	/* next */
5095	Assert(cb > cbWrite);
5096	cb -= cbWrite;
5097	pvSrc = (uint8_t *)pvSrc + cbWrite;
5098	GCPtrDst += cbWrite;
5099	}
5100	}
5101
5102
5103	/**
5104	* Return the page type of the specified physical address.
5105	*
5106	* @returns The page type.
5107	* @param pVM The cross context VM structure.
5108	* @param GCPhys Guest physical address
5109	*/
5110	VMM_INT_DECL(PGMPAGETYPE) PGMPhysGetPageType(PVMCC pVM, RTGCPHYS GCPhys)
5111	{
5112	PGM_LOCK_VOID(pVM);
5113	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
5114	PGMPAGETYPE enmPgType = pPage ? (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage) : PGMPAGETYPE_INVALID;
5115	PGM_UNLOCK(pVM);
5116
5117	return enmPgType;
5118	}
5119
5120
5121	/** Helper for PGMPhysIemGCPhys2PtrNoLock. */
5122	DECL_FORCE_INLINE(int)
5123	pgmPhyIemGCphys2PtrNoLockReturnNoNothing(uint64_t uTlbPhysRev, R3R0PTRTYPE(uint8_t ) ppb, uint64_t *pfTlb,
5124	RTGCPHYS GCPhys, PCPGMPAGE pPageCopy)
5125	{
5126	*pfTlb \|= uTlbPhysRev
5127	\| PGMIEMGCPHYS2PTR_F_NO_WRITE \| PGMIEMGCPHYS2PTR_F_NO_READ \| PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
5128	*ppb = NULL;
5129	Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp ppb=NULL pfTlb=%#RX64 PageCopy=%R[pgmpage] NO\n", GCPhys,
5130	uTlbPhysRev \| PGMIEMGCPHYS2PTR_F_NO_WRITE \| PGMIEMGCPHYS2PTR_F_NO_READ \| PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3, pPageCopy));
5131	RT_NOREF(GCPhys, pPageCopy);
5132	return VINF_SUCCESS;
5133	}
5134
5135
5136	/** Helper for PGMPhysIemGCPhys2PtrNoLock. */
5137	DECL_FORCE_INLINE(int)
5138	pgmPhyIemGCphys2PtrNoLockReturnReadOnly(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uTlbPhysRev, RTGCPHYS GCPhys, PCPGMPAGE pPageCopy,
5139	PPGMRAMRANGE pRam, PPGMPAGE pPage, R3R0PTRTYPE(uint8_t ) ppb, uint64_t *pfTlb)
5140	{
5141	RT_NOREF(GCPhys);
5142	if (!PGM_PAGE_IS_CODE_PAGE(pPageCopy))
5143	*pfTlb \|= uTlbPhysRev \| PGMIEMGCPHYS2PTR_F_NO_WRITE;
5144	else
5145	*pfTlb \|= uTlbPhysRev \| PGMIEMGCPHYS2PTR_F_NO_WRITE \| PGMIEMGCPHYS2PTR_F_CODE_PAGE;
5146
5147	#ifdef IN_RING3
5148	if (PGM_IS_IN_NEM_MODE(pVM))
5149	*ppb = &pRam->pbR3[(RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << GUEST_PAGE_SHIFT];
5150	else
5151	#endif
5152	{
5153	#ifdef IN_RING3
5154	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
5155	PPGMPAGEMAPTLBE pTlbe;
5156	int rc = pgmPhysPageQueryLocklessTlbeWithPage(pVCpu, pPage, GCPhys, &pTlbe);
5157	AssertLogRelRCReturn(rc, rc);
5158	ppb = (uint8_t )pTlbe->pv;
5159	RT_NOREF(pVM);
5160	# endif
5161	#else /** @todo a safe lockless page TLB in ring-0 needs the to ensure it gets the right invalidations. later. */
5162	PGM_LOCK(pVM);
5163	PPGMPAGEMAPTLBE pTlbe;
5164	int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
5165	AssertLogRelRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
5166	ppb = (uint8_t )pTlbe->pv;
5167	PGM_UNLOCK(pVM);
5168	RT_NOREF(pVCpu);
5169	#endif
5170	}
5171	Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp ppb=%p pfTlb=%#RX64 PageCopy=%R[pgmpage] RO\n", GCPhys, ppb, pfTlb, pPageCopy));
5172	RT_NOREF(pRam, pVM, pVCpu);
5173	return VINF_SUCCESS;
5174	}
5175
5176
5177	/** Helper for PGMPhysIemGCPhys2PtrNoLock. */
5178	DECL_FORCE_INLINE(int)
5179	pgmPhyIemGCphys2PtrNoLockReturnReadWrite(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uTlbPhysRev, RTGCPHYS GCPhys, PCPGMPAGE pPageCopy,
5180	PPGMRAMRANGE pRam, PPGMPAGE pPage, R3R0PTRTYPE(uint8_t ) ppb, uint64_t *pfTlb)
5181	{
5182	Assert(!PGM_PAGE_IS_CODE_PAGE(pPageCopy));
5183	RT_NOREF(pPageCopy, GCPhys);
5184	*pfTlb \|= uTlbPhysRev;
5185
5186	#ifdef IN_RING3
5187	if (PGM_IS_IN_NEM_MODE(pVM))
5188	*ppb = &pRam->pbR3[(RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << GUEST_PAGE_SHIFT];
5189	else
5190	#endif
5191	{
5192	#ifdef IN_RING3
5193	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
5194	PPGMPAGEMAPTLBE pTlbe;
5195	int rc = pgmPhysPageQueryLocklessTlbeWithPage(pVCpu, pPage, GCPhys, &pTlbe);
5196	AssertLogRelRCReturn(rc, rc);
5197	ppb = (uint8_t )pTlbe->pv;
5198	RT_NOREF(pVM);
5199	# endif
5200	#else /** @todo a safe lockless page TLB in ring-0 needs the to ensure it gets the right invalidations. later. */
5201	PGM_LOCK(pVM);
5202	PPGMPAGEMAPTLBE pTlbe;
5203	int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
5204	AssertLogRelRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
5205	ppb = (uint8_t )pTlbe->pv;
5206	PGM_UNLOCK(pVM);
5207	RT_NOREF(pVCpu);
5208	#endif
5209	}
5210	Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp ppb=%p pfTlb=%#RX64 PageCopy=%R[pgmpage] RW\n", GCPhys, ppb, pfTlb, pPageCopy));
5211	RT_NOREF(pRam, pVM, pVCpu);
5212	return VINF_SUCCESS;
5213	}
5214
5215
5216	/**
5217	* Converts a GC physical address to a HC ring-3 pointer, with some
5218	* additional checks.
5219	*
5220	* @returns VBox status code (no informational statuses).
5221	*
5222	* @param pVM The cross context VM structure.
5223	* @param pVCpu The cross context virtual CPU structure of the
5224	* calling EMT.
5225	* @param GCPhys The GC physical address to convert. This API mask
5226	* the A20 line when necessary.
5227	* @param puTlbPhysRev Where to read the physical TLB revision. Needs to
5228	* be done while holding the PGM lock.
5229	* @param ppb Where to store the pointer corresponding to GCPhys
5230	* on success.
5231	* @param pfTlb The TLB flags and revision. We only add stuff.
5232	*
5233	* @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr and
5234	* PGMPhysIemGCPhys2Ptr.
5235	*
5236	* @thread EMT(pVCpu).
5237	*/
5238	VMM_INT_DECL(int) PGMPhysIemGCPhys2PtrNoLock(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint64_t const volatile *puTlbPhysRev,
5239	R3R0PTRTYPE(uint8_t ) ppb, uint64_t *pfTlb)
5240	{
5241	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
5242	Assert(!(GCPhys & X86_PAGE_OFFSET_MASK));
5243
5244	PGMRAMRANGE volatile *pRam;
5245	PGMPAGE volatile *pPage;
5246	int rc = pgmPhysGetPageAndRangeExLockless(pVM, pVCpu, GCPhys, &pPage, &pRam);
5247	if (RT_SUCCESS(rc))
5248	{
5249	/*
5250	* Wrt to update races, we will try to pretend we beat the update we're
5251	* racing. We do this by sampling the physical TLB revision first, so
5252	* that the TLB entry / whatever purpose the caller has with the info
5253	* will become invalid immediately if it's updated.
5254	*
5255	* This means the caller will (probably) make use of the returned info
5256	* only once and then requery it the next time it is use, getting the
5257	* updated info. This would then be just as if the first query got the
5258	* PGM lock before the updater.
5259	*/
5260	/** @todo make PGMPAGE updates more atomic, possibly flagging complex
5261	* updates by adding a u1UpdateInProgress field (or revision).
5262	* This would be especially important when updating the page ID... */
5263	uint64_t uTlbPhysRev = *puTlbPhysRev;
5264	PGMPAGE PageCopy = { { pPage->au64[0], pPage->au64[1] } };
5265	if ( uTlbPhysRev == *puTlbPhysRev
5266	&& PageCopy.au64[0] == pPage->au64[0]
5267	&& PageCopy.au64[1] == pPage->au64[1])
5268	ASMCompilerBarrier(); /* likely */
5269	else
5270	{
5271	PGM_LOCK_VOID(pVM);
5272	uTlbPhysRev = *puTlbPhysRev;
5273	PageCopy.au64[0] = pPage->au64[0];
5274	PageCopy.au64[1] = pPage->au64[1];
5275	PGM_UNLOCK(pVM);
5276	}
5277
5278	/*
5279	* Try optimize for the regular case first: Writable RAM.
5280	*/
5281	switch (PGM_PAGE_GET_HNDL_PHYS_STATE(&PageCopy))
5282	{
5283	case PGM_PAGE_HNDL_PHYS_STATE_DISABLED:
5284	if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(&PageCopy))
5285	{ /* likely */ }
5286	else
5287	return pgmPhyIemGCphys2PtrNoLockReturnNoNothing(uTlbPhysRev, ppb, pfTlb, GCPhys, &PageCopy);
5288	RT_FALL_THRU();
5289	case PGM_PAGE_HNDL_PHYS_STATE_NONE:
5290	Assert(!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(&PageCopy));
5291	switch (PGM_PAGE_GET_STATE_NA(&PageCopy))
5292	{
5293	case PGM_PAGE_STATE_ALLOCATED:
5294	return pgmPhyIemGCphys2PtrNoLockReturnReadWrite(pVM, pVCpu, uTlbPhysRev, GCPhys, &PageCopy,
5295	(PPGMRAMRANGE)pRam, (PPGMPAGE)pPage, ppb, pfTlb);
5296
5297	case PGM_PAGE_STATE_ZERO:
5298	case PGM_PAGE_STATE_WRITE_MONITORED:
5299	case PGM_PAGE_STATE_SHARED:
5300	return pgmPhyIemGCphys2PtrNoLockReturnReadOnly(pVM, pVCpu, uTlbPhysRev, GCPhys, &PageCopy,
5301	(PPGMRAMRANGE)pRam, (PPGMPAGE)pPage, ppb, pfTlb);
5302
5303	default: AssertFailed(); RT_FALL_THROUGH();
5304	case PGM_PAGE_STATE_BALLOONED:
5305	return pgmPhyIemGCphys2PtrNoLockReturnNoNothing(uTlbPhysRev, ppb, pfTlb, GCPhys, &PageCopy);
5306	}
5307	break;
5308
5309	case PGM_PAGE_HNDL_PHYS_STATE_WRITE:
5310	Assert(!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(&PageCopy));
5311	switch (PGM_PAGE_GET_STATE_NA(&PageCopy))
5312	{
5313	case PGM_PAGE_STATE_ALLOCATED:
5314	Assert(!PGM_PAGE_IS_CODE_PAGE(&PageCopy));
5315	RT_FALL_THRU();
5316	case PGM_PAGE_STATE_ZERO:
5317	case PGM_PAGE_STATE_WRITE_MONITORED:
5318	case PGM_PAGE_STATE_SHARED:
5319	return pgmPhyIemGCphys2PtrNoLockReturnReadOnly(pVM, pVCpu, uTlbPhysRev, GCPhys, &PageCopy,
5320	(PPGMRAMRANGE)pRam, (PPGMPAGE)pPage, ppb, pfTlb);
5321
5322	default: AssertFailed(); RT_FALL_THROUGH();
5323	case PGM_PAGE_STATE_BALLOONED:
5324	return pgmPhyIemGCphys2PtrNoLockReturnNoNothing(uTlbPhysRev, ppb, pfTlb, GCPhys, &PageCopy);
5325	}
5326	break;
5327
5328	case PGM_PAGE_HNDL_PHYS_STATE_ALL:
5329	Assert(!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(&PageCopy));
5330	return pgmPhyIemGCphys2PtrNoLockReturnNoNothing(uTlbPhysRev, ppb, pfTlb, GCPhys, &PageCopy);
5331	}
5332	}
5333	else
5334	{
5335	pfTlb \|= puTlbPhysRev \| PGMIEMGCPHYS2PTR_F_NO_WRITE \| PGMIEMGCPHYS2PTR_F_NO_READ
5336	\| PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 \| PGMIEMGCPHYS2PTR_F_UNASSIGNED;
5337	*ppb = NULL;
5338	Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp ppb=%p pfTlb=%#RX64 (rc=%Rrc)\n", GCPhys, ppb, pfTlb, rc));
5339	}
5340
5341	return VINF_SUCCESS;
5342	}
5343
5344
5345	/**
5346	* Converts a GC physical address to a HC ring-3 pointer, with some
5347	* additional checks.
5348	*
5349	* @returns VBox status code (no informational statuses).
5350	* @retval VINF_SUCCESS on success.
5351	* @retval VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
5352	* access handler of some kind.
5353	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
5354	* accesses or is odd in any way.
5355	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
5356	*
5357	* @param pVM The cross context VM structure.
5358	* @param pVCpu The cross context virtual CPU structure of the
5359	* calling EMT.
5360	* @param GCPhys The GC physical address to convert. This API mask
5361	* the A20 line when necessary.
5362	* @param fWritable Whether write access is required.
5363	* @param fByPassHandlers Whether to bypass access handlers.
5364	* @param ppv Where to store the pointer corresponding to GCPhys
5365	* on success.
5366	* @param pLock
5367	*
5368	* @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr.
5369	* @thread EMT(pVCpu).
5370	*/
5371	VMM_INT_DECL(int) PGMPhysIemGCPhys2Ptr(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers,
5372	void **ppv, PPGMPAGEMAPLOCK pLock)
5373	{
5374	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
5375
5376	PGM_LOCK_VOID(pVM);
5377
5378	PPGMRAMRANGE pRam;
5379	PPGMPAGE pPage;
5380	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
5381	if (RT_SUCCESS(rc))
5382	{
5383	if (PGM_PAGE_IS_BALLOONED(pPage))
5384	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
5385	else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
5386	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
5387	else if ( !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
5388	\|\| (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
5389	rc = VINF_SUCCESS;
5390	else
5391	{
5392	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
5393	{
5394	Assert(!fByPassHandlers \|\| PGM_PAGE_IS_MMIO(pPage));
5395	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
5396	}
5397	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
5398	{
5399	Assert(!fByPassHandlers);
5400	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
5401	}
5402	}
5403	if (RT_SUCCESS(rc))
5404	{
5405	int rc2;
5406
5407	/* Make sure what we return is writable. */
5408	if (fWritable)
5409	switch (PGM_PAGE_GET_STATE(pPage))
5410	{
5411	case PGM_PAGE_STATE_ALLOCATED:
5412	break;
5413	case PGM_PAGE_STATE_BALLOONED:
5414	AssertFailed();
5415	break;
5416	case PGM_PAGE_STATE_ZERO:
5417	case PGM_PAGE_STATE_SHARED:
5418	case PGM_PAGE_STATE_WRITE_MONITORED:
5419	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
5420	AssertLogRelRCReturn(rc2, rc2);
5421	break;
5422	}
5423
5424	/* Get a ring-3 mapping of the address. */
5425	PPGMPAGEMAPTLBE pTlbe;
5426	rc2 = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
5427	AssertLogRelRCReturn(rc2, rc2);
5428
5429	/* Lock it and calculate the address. */
5430	if (fWritable)
5431	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
5432	else
5433	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
5434	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
5435
5436	Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
5437	}
5438	else
5439	Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
5440
5441	/* else: handler catching all access, no pointer returned. */
5442	}
5443	else
5444	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
5445
5446	PGM_UNLOCK(pVM);
5447	return rc;
5448	}
5449
5450
5451	/**
5452	* Checks if the give GCPhys page requires special handling for the given access
5453	* because it's MMIO or otherwise monitored.
5454	*
5455	* @returns VBox status code (no informational statuses).
5456	* @retval VINF_SUCCESS on success.
5457	* @retval VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
5458	* access handler of some kind.
5459	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
5460	* accesses or is odd in any way.
5461	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
5462	*
5463	* @param pVM The cross context VM structure.
5464	* @param GCPhys The GC physical address to convert. Since this is
5465	* only used for filling the REM TLB, the A20 mask must
5466	* be applied before calling this API.
5467	* @param fWritable Whether write access is required.
5468	* @param fByPassHandlers Whether to bypass access handlers.
5469	*
5470	* @remarks This is a watered down version PGMPhysIemGCPhys2Ptr and really just
5471	* a stop gap thing that should be removed once there is a better TLB
5472	* for virtual address accesses.
5473	*/
5474	VMM_INT_DECL(int) PGMPhysIemQueryAccess(PVMCC pVM, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers)
5475	{
5476	PGM_LOCK_VOID(pVM);
5477	PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
5478
5479	PPGMRAMRANGE pRam;
5480	PPGMPAGE pPage;
5481	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
5482	if (RT_SUCCESS(rc))
5483	{
5484	if (PGM_PAGE_IS_BALLOONED(pPage))
5485	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
5486	else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
5487	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
5488	else if ( !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
5489	\|\| (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
5490	rc = VINF_SUCCESS;
5491	else
5492	{
5493	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
5494	{
5495	Assert(!fByPassHandlers \|\| PGM_PAGE_IS_MMIO(pPage));
5496	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
5497	}
5498	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
5499	{
5500	Assert(!fByPassHandlers);
5501	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
5502	}
5503	}
5504	}
5505
5506	PGM_UNLOCK(pVM);
5507	return rc;
5508	}
5509
5510	#ifdef VBOX_WITH_NATIVE_NEM
5511
5512	/**
5513	* Interface used by NEM to check what to do on a memory access exit.
5514	*
5515	* @returns VBox status code.
5516	* @param pVM The cross context VM structure.
5517	* @param pVCpu The cross context per virtual CPU structure.
5518	* Optional.
5519	* @param GCPhys The guest physical address.
5520	* @param fMakeWritable Whether to try make the page writable or not. If it
5521	* cannot be made writable, NEM_PAGE_PROT_WRITE won't
5522	* be returned and the return code will be unaffected
5523	* @param pInfo Where to return the page information. This is
5524	* initialized even on failure.
5525	* @param pfnChecker Page in-sync checker callback. Optional.
5526	* @param pvUser User argument to pass to pfnChecker.
5527	*/
5528	VMM_INT_DECL(int) PGMPhysNemPageInfoChecker(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, bool fMakeWritable,
5529	PPGMPHYSNEMPAGEINFO pInfo, PFNPGMPHYSNEMCHECKPAGE pfnChecker, void *pvUser)
5530	{
5531	PGM_LOCK_VOID(pVM);
5532
5533	PPGMPAGE pPage;
5534	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
5535	if (RT_SUCCESS(rc))
5536	{
5537	/* Try make it writable if requested. */
5538	pInfo->u2OldNemState = PGM_PAGE_GET_NEM_STATE(pPage);
5539	if (fMakeWritable)
5540	switch (PGM_PAGE_GET_STATE(pPage))
5541	{
5542	case PGM_PAGE_STATE_SHARED:
5543	case PGM_PAGE_STATE_WRITE_MONITORED:
5544	case PGM_PAGE_STATE_ZERO:
5545	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
5546	if (rc == VERR_PGM_PHYS_PAGE_RESERVED)
5547	rc = VINF_SUCCESS;
5548	break;
5549	}
5550
5551	/* Fill in the info. */
5552	pInfo->HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
5553	pInfo->u2NemState = PGM_PAGE_GET_NEM_STATE(pPage);
5554	pInfo->fHasHandlers = PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) ? 1 : 0;
5555	PGMPAGETYPE const enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
5556	pInfo->enmType = enmType;
5557	pInfo->fNemProt = pgmPhysPageCalcNemProtection(pPage, enmType);
5558	switch (PGM_PAGE_GET_STATE(pPage))
5559	{
5560	case PGM_PAGE_STATE_ALLOCATED:
5561	pInfo->fZeroPage = 0;
5562	break;
5563
5564	case PGM_PAGE_STATE_ZERO:
5565	pInfo->fZeroPage = 1;
5566	break;
5567
5568	case PGM_PAGE_STATE_WRITE_MONITORED:
5569	pInfo->fZeroPage = 0;
5570	break;
5571
5572	case PGM_PAGE_STATE_SHARED:
5573	pInfo->fZeroPage = 0;
5574	break;
5575
5576	case PGM_PAGE_STATE_BALLOONED:
5577	pInfo->fZeroPage = 1;
5578	break;
5579
5580	default:
5581	pInfo->fZeroPage = 1;
5582	AssertFailedStmt(rc = VERR_PGM_PHYS_PAGE_GET_IPE);
5583	}
5584
5585	/* Call the checker and update NEM state. */
5586	if (pfnChecker)
5587	{
5588	rc = pfnChecker(pVM, pVCpu, GCPhys, pInfo, pvUser);
5589	PGM_PAGE_SET_NEM_STATE(pPage, pInfo->u2NemState);
5590	}
5591
5592	/* Done. */
5593	PGM_UNLOCK(pVM);
5594	}
5595	else
5596	{
5597	PGM_UNLOCK(pVM);
5598
5599	pInfo->HCPhys = NIL_RTHCPHYS;
5600	pInfo->fNemProt = NEM_PAGE_PROT_NONE;
5601	pInfo->u2NemState = 0;
5602	pInfo->fHasHandlers = 0;
5603	pInfo->fZeroPage = 0;
5604	pInfo->enmType = PGMPAGETYPE_INVALID;
5605	}
5606
5607	return rc;
5608	}
5609
5610
5611	/**
5612	* NEM helper that performs @a pfnCallback on pages with NEM state @a uMinState
5613	* or higher.
5614	*
5615	* @returns VBox status code from callback.
5616	* @param pVM The cross context VM structure.
5617	* @param pVCpu The cross context per CPU structure. This is
5618	* optional as its only for passing to callback.
5619	* @param uMinState The minimum NEM state value to call on.
5620	* @param pfnCallback The callback function.
5621	* @param pvUser User argument for the callback.
5622	*/
5623	VMM_INT_DECL(int) PGMPhysNemEnumPagesByState(PVMCC pVM, PVMCPUCC pVCpu, uint8_t uMinState,
5624	PFNPGMPHYSNEMENUMCALLBACK pfnCallback, void *pvUser)
5625	{
5626	/*
5627	* Just brute force this problem.
5628	*/
5629	PGM_LOCK_VOID(pVM);
5630	int rc = VINF_SUCCESS;
5631	uint32_t const cLookupEntries = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
5632	for (uint32_t idxLookup = 0; idxLookup < cLookupEntries && RT_SUCCESS(rc); idxLookup++)
5633	{
5634	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
5635	AssertContinue(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges));
5636	PPGMRAMRANGE const pRam = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
5637	AssertContinue(pRam);
5638	Assert(pRam->GCPhys == PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]));
5639
5640	#ifdef IN_RING0
5641	uint32_t const cPages = RT_MIN(pRam->cb >> X86_PAGE_SHIFT, pVM->pgmr0.s.acRamRangePages[idRamRange]);
5642	#else
5643	uint32_t const cPages = pRam->cb >> X86_PAGE_SHIFT;
5644	#endif
5645	for (uint32_t iPage = 0; iPage < cPages; iPage++)
5646	{
5647	uint8_t u2State = PGM_PAGE_GET_NEM_STATE(&pRam->aPages[iPage]);
5648	if (u2State < uMinState)
5649	{ /* likely */ }
5650	else
5651	{
5652	rc = pfnCallback(pVM, pVCpu, pRam->GCPhys + ((RTGCPHYS)iPage << X86_PAGE_SHIFT), &u2State, pvUser);
5653	if (RT_SUCCESS(rc))
5654	PGM_PAGE_SET_NEM_STATE(&pRam->aPages[iPage], u2State);
5655	else
5656	break;
5657	}
5658	}
5659	}
5660	PGM_UNLOCK(pVM);
5661
5662	return rc;
5663	}
5664
5665
5666	/**
5667	* Helper for setting the NEM state for a range of pages.
5668	*
5669	* @param paPages Array of pages to modify.
5670	* @param cPages How many pages to modify.
5671	* @param u2State The new state value.
5672	*/
5673	DECLHIDDEN(void) pgmPhysSetNemStateForPages(PPGMPAGE paPages, RTGCPHYS cPages, uint8_t u2State)
5674	{
5675	PPGMPAGE pPage = paPages;
5676	while (cPages-- > 0)
5677	{
5678	PGM_PAGE_SET_NEM_STATE(pPage, u2State);
5679	pPage++;
5680	}
5681	}
5682
5683	#endif /* VBOX_WITH_NATIVE_NEM */
5684

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp@ 107631

Download in other formats: