PGMPhys.cpp@ 80182

Last change on this file since 80182 was 80182, checked in by vboxsync, 5 years ago
VMM: Kicking out raw-mode - Eliminated more RCPTRTYPE use in PGM. bugref:9517
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 206.6 KB

Line
1	/* $Id: PGMPhys.cpp 80182 2019-08-07 11:17:11Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_PHYS
23	#include <VBox/vmm/pgm.h>
24	#include <VBox/vmm/iem.h>
25	#include <VBox/vmm/iom.h>
26	#include <VBox/vmm/mm.h>
27	#include <VBox/vmm/nem.h>
28	#include <VBox/vmm/stam.h>
29	#ifdef VBOX_WITH_REM
30	# include <VBox/vmm/rem.h>
31	#endif
32	#include <VBox/vmm/pdmdev.h>
33	#include "PGMInternal.h"
34	#include <VBox/vmm/vm.h>
35	#include <VBox/vmm/uvm.h>
36	#include "PGMInline.h"
37	#include <VBox/sup.h>
38	#include <VBox/param.h>
39	#include <VBox/err.h>
40	#include <VBox/log.h>
41	#include <iprt/assert.h>
42	#include <iprt/alloc.h>
43	#include <iprt/asm.h>
44	#ifdef VBOX_STRICT
45	# include <iprt/crc.h>
46	#endif
47	#include <iprt/thread.h>
48	#include <iprt/string.h>
49	#include <iprt/system.h>
50
51
52	/*********************************************************************************************************************************
53	* Defined Constants And Macros *
54	*********************************************************************************************************************************/
55	/** The number of pages to free in one batch. */
56	#define PGMPHYS_FREE_PAGE_BATCH_SIZE 128
57
58
59	/*
60	* PGMR3PhysReadU8-64
61	* PGMR3PhysWriteU8-64
62	*/
63	#define PGMPHYSFN_READNAME PGMR3PhysReadU8
64	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
65	#define PGMPHYS_DATASIZE 1
66	#define PGMPHYS_DATATYPE uint8_t
67	#include "PGMPhysRWTmpl.h"
68
69	#define PGMPHYSFN_READNAME PGMR3PhysReadU16
70	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
71	#define PGMPHYS_DATASIZE 2
72	#define PGMPHYS_DATATYPE uint16_t
73	#include "PGMPhysRWTmpl.h"
74
75	#define PGMPHYSFN_READNAME PGMR3PhysReadU32
76	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
77	#define PGMPHYS_DATASIZE 4
78	#define PGMPHYS_DATATYPE uint32_t
79	#include "PGMPhysRWTmpl.h"
80
81	#define PGMPHYSFN_READNAME PGMR3PhysReadU64
82	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
83	#define PGMPHYS_DATASIZE 8
84	#define PGMPHYS_DATATYPE uint64_t
85	#include "PGMPhysRWTmpl.h"
86
87
88	/**
89	* EMT worker for PGMR3PhysReadExternal.
90	*/
91	static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead,
92	PGMACCESSORIGIN enmOrigin)
93	{
94	VBOXSTRICTRC rcStrict = PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead, enmOrigin);
95	AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); NOREF(rcStrict);
96	return VINF_SUCCESS;
97	}
98
99
100	/**
101	* Read from physical memory, external users.
102	*
103	* @returns VBox status code.
104	* @retval VINF_SUCCESS.
105	*
106	* @param pVM The cross context VM structure.
107	* @param GCPhys Physical address to read from.
108	* @param pvBuf Where to read into.
109	* @param cbRead How many bytes to read.
110	* @param enmOrigin Who is calling.
111	*
112	* @thread Any but EMTs.
113	*/
114	VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead, PGMACCESSORIGIN enmOrigin)
115	{
116	VM_ASSERT_OTHER_THREAD(pVM);
117
118	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
119	LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
120
121	pgmLock(pVM);
122
123	/*
124	* Copy loop on ram ranges.
125	*/
126	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
127	for (;;)
128	{
129	/* Inside range or not? */
130	if (pRam && GCPhys >= pRam->GCPhys)
131	{
132	/*
133	* Must work our way thru this page by page.
134	*/
135	RTGCPHYS off = GCPhys - pRam->GCPhys;
136	while (off < pRam->cb)
137	{
138	unsigned iPage = off >> PAGE_SHIFT;
139	PPGMPAGE pPage = &pRam->aPages[iPage];
140
141	/*
142	* If the page has an ALL access handler, we'll have to
143	* delegate the job to EMT.
144	*/
145	if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
146	\|\| PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
147	{
148	pgmUnlock(pVM);
149
150	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 5,
151	pVM, &GCPhys, pvBuf, cbRead, enmOrigin);
152	}
153	Assert(!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage));
154
155	/*
156	* Simple stuff, go ahead.
157	*/
158	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
159	if (cb > cbRead)
160	cb = cbRead;
161	PGMPAGEMAPLOCK PgMpLck;
162	const void *pvSrc;
163	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
164	if (RT_SUCCESS(rc))
165	{
166	memcpy(pvBuf, pvSrc, cb);
167	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
168	}
169	else
170	{
171	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
172	pRam->GCPhys + off, pPage, rc));
173	memset(pvBuf, 0xff, cb);
174	}
175
176	/* next page */
177	if (cb >= cbRead)
178	{
179	pgmUnlock(pVM);
180	return VINF_SUCCESS;
181	}
182	cbRead -= cb;
183	off += cb;
184	GCPhys += cb;
185	pvBuf = (char *)pvBuf + cb;
186	} /* walk pages in ram range. */
187	}
188	else
189	{
190	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
191
192	/*
193	* Unassigned address space.
194	*/
195	size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
196	if (cb >= cbRead)
197	{
198	memset(pvBuf, 0xff, cbRead);
199	break;
200	}
201	memset(pvBuf, 0xff, cb);
202
203	cbRead -= cb;
204	pvBuf = (char *)pvBuf + cb;
205	GCPhys += cb;
206	}
207
208	/* Advance range if necessary. */
209	while (pRam && GCPhys > pRam->GCPhysLast)
210	pRam = pRam->CTX_SUFF(pNext);
211	} /* Ram range walk */
212
213	pgmUnlock(pVM);
214
215	return VINF_SUCCESS;
216	}
217
218
219	/**
220	* EMT worker for PGMR3PhysWriteExternal.
221	*/
222	static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite,
223	PGMACCESSORIGIN enmOrigin)
224	{
225	/** @todo VERR_EM_NO_MEMORY */
226	VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite, enmOrigin);
227	AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); NOREF(rcStrict);
228	return VINF_SUCCESS;
229	}
230
231
232	/**
233	* Write to physical memory, external users.
234	*
235	* @returns VBox status code.
236	* @retval VINF_SUCCESS.
237	* @retval VERR_EM_NO_MEMORY.
238	*
239	* @param pVM The cross context VM structure.
240	* @param GCPhys Physical address to write to.
241	* @param pvBuf What to write.
242	* @param cbWrite How many bytes to write.
243	* @param enmOrigin Who is calling.
244	*
245	* @thread Any but EMTs.
246	*/
247	VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, PGMACCESSORIGIN enmOrigin)
248	{
249	VM_ASSERT_OTHER_THREAD(pVM);
250
251	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
252	("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x enmOrigin=%d\n",
253	GCPhys, cbWrite, enmOrigin));
254	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
255	LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
256
257	pgmLock(pVM);
258
259	/*
260	* Copy loop on ram ranges, stop when we hit something difficult.
261	*/
262	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
263	for (;;)
264	{
265	/* Inside range or not? */
266	if (pRam && GCPhys >= pRam->GCPhys)
267	{
268	/*
269	* Must work our way thru this page by page.
270	*/
271	RTGCPTR off = GCPhys - pRam->GCPhys;
272	while (off < pRam->cb)
273	{
274	RTGCPTR iPage = off >> PAGE_SHIFT;
275	PPGMPAGE pPage = &pRam->aPages[iPage];
276
277	/*
278	* Is the page problematic, we have to do the work on the EMT.
279	*
280	* Allocating writable pages and access handlers are
281	* problematic, write monitored pages are simple and can be
282	* dealt with here.
283	*/
284	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
285	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
286	\|\| PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
287	{
288	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
289	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
290	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
291	else
292	{
293	pgmUnlock(pVM);
294
295	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 5,
296	pVM, &GCPhys, pvBuf, cbWrite, enmOrigin);
297	}
298	}
299	Assert(!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage));
300
301	/*
302	* Simple stuff, go ahead.
303	*/
304	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
305	if (cb > cbWrite)
306	cb = cbWrite;
307	PGMPAGEMAPLOCK PgMpLck;
308	void *pvDst;
309	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
310	if (RT_SUCCESS(rc))
311	{
312	memcpy(pvDst, pvBuf, cb);
313	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
314	}
315	else
316	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
317	pRam->GCPhys + off, pPage, rc));
318
319	/* next page */
320	if (cb >= cbWrite)
321	{
322	pgmUnlock(pVM);
323	return VINF_SUCCESS;
324	}
325
326	cbWrite -= cb;
327	off += cb;
328	GCPhys += cb;
329	pvBuf = (const char *)pvBuf + cb;
330	} /* walk pages in ram range */
331	}
332	else
333	{
334	/*
335	* Unassigned address space, skip it.
336	*/
337	if (!pRam)
338	break;
339	size_t cb = pRam->GCPhys - GCPhys;
340	if (cb >= cbWrite)
341	break;
342	cbWrite -= cb;
343	pvBuf = (const char *)pvBuf + cb;
344	GCPhys += cb;
345	}
346
347	/* Advance range if necessary. */
348	while (pRam && GCPhys > pRam->GCPhysLast)
349	pRam = pRam->CTX_SUFF(pNext);
350	} /* Ram range walk */
351
352	pgmUnlock(pVM);
353	return VINF_SUCCESS;
354	}
355
356
357	/**
358	* VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
359	*
360	* @returns see PGMR3PhysGCPhys2CCPtrExternal
361	* @param pVM The cross context VM structure.
362	* @param pGCPhys Pointer to the guest physical address.
363	* @param ppv Where to store the mapping address.
364	* @param pLock Where to store the lock.
365	*/
366	static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
367	{
368	/*
369	* Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
370	* an access handler after it succeeds.
371	*/
372	int rc = pgmLock(pVM);
373	AssertRCReturn(rc, rc);
374
375	rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
376	if (RT_SUCCESS(rc))
377	{
378	PPGMPAGEMAPTLBE pTlbe;
379	int rc2 = pgmPhysPageQueryTlbe(pVM, *pGCPhys, &pTlbe);
380	AssertFatalRC(rc2);
381	PPGMPAGE pPage = pTlbe->pPage;
382	if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
383	{
384	PGMPhysReleasePageMappingLock(pVM, pLock);
385	rc = VERR_PGM_PHYS_PAGE_RESERVED;
386	}
387	else if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
388	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
389	\|\| pgmPoolIsDirtyPage(pVM, *pGCPhys)
390	#endif
391	)
392	{
393	/* We must flush any corresponding pgm pool page here, otherwise we'll
394	* not be informed about writes and keep bogus gst->shw mappings around.
395	*/
396	pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
397	Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
398	/** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
399	* active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
400	}
401	}
402
403	pgmUnlock(pVM);
404	return rc;
405	}
406
407
408	/**
409	* Requests the mapping of a guest page into ring-3, external threads.
410	*
411	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
412	* release it.
413	*
414	* This API will assume your intention is to write to the page, and will
415	* therefore replace shared and zero pages. If you do not intend to modify the
416	* page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
417	*
418	* @returns VBox status code.
419	* @retval VINF_SUCCESS on success.
420	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
421	* backing or if the page has any active access handlers. The caller
422	* must fall back on using PGMR3PhysWriteExternal.
423	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
424	*
425	* @param pVM The cross context VM structure.
426	* @param GCPhys The guest physical address of the page that should be mapped.
427	* @param ppv Where to store the address corresponding to GCPhys.
428	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
429	*
430	* @remark Avoid calling this API from within critical sections (other than the
431	* PGM one) because of the deadlock risk when we have to delegating the
432	* task to an EMT.
433	* @thread Any.
434	*/
435	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
436	{
437	AssertPtr(ppv);
438	AssertPtr(pLock);
439
440	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
441
442	int rc = pgmLock(pVM);
443	AssertRCReturn(rc, rc);
444
445	/*
446	* Query the Physical TLB entry for the page (may fail).
447	*/
448	PPGMPAGEMAPTLBE pTlbe;
449	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
450	if (RT_SUCCESS(rc))
451	{
452	PPGMPAGE pPage = pTlbe->pPage;
453	if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
454	rc = VERR_PGM_PHYS_PAGE_RESERVED;
455	else
456	{
457	/*
458	* If the page is shared, the zero page, or being write monitored
459	* it must be converted to an page that's writable if possible.
460	* We can only deal with write monitored pages here, the rest have
461	* to be on an EMT.
462	*/
463	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
464	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
465	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
466	\|\| pgmPoolIsDirtyPage(pVM, GCPhys)
467	#endif
468	)
469	{
470	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
471	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
472	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
473	&& !pgmPoolIsDirtyPage(pVM, GCPhys) /** @todo we're very likely doing this twice. */
474	#endif
475	)
476	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
477	else
478	{
479	pgmUnlock(pVM);
480
481	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
482	pVM, &GCPhys, ppv, pLock);
483	}
484	}
485
486	/*
487	* Now, just perform the locking and calculate the return address.
488	*/
489	PPGMPAGEMAP pMap = pTlbe->pMap;
490	if (pMap)
491	pMap->cRefs++;
492
493	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
494	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
495	{
496	if (cLocks == 0)
497	pVM->pgm.s.cWriteLockedPages++;
498	PGM_PAGE_INC_WRITE_LOCKS(pPage);
499	}
500	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
501	{
502	PGM_PAGE_INC_WRITE_LOCKS(pPage);
503	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
504	if (pMap)
505	pMap->cRefs++; /* Extra ref to prevent it from going away. */
506	}
507
508	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
509	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
510	pLock->pvMap = pMap;
511	}
512	}
513
514	pgmUnlock(pVM);
515	return rc;
516	}
517
518
519	/**
520	* Requests the mapping of a guest page into ring-3, external threads.
521	*
522	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
523	* release it.
524	*
525	* @returns VBox status code.
526	* @retval VINF_SUCCESS on success.
527	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
528	* backing or if the page as an active ALL access handler. The caller
529	* must fall back on using PGMPhysRead.
530	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
531	*
532	* @param pVM The cross context VM structure.
533	* @param GCPhys The guest physical address of the page that should be mapped.
534	* @param ppv Where to store the address corresponding to GCPhys.
535	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
536	*
537	* @remark Avoid calling this API from within critical sections (other than
538	* the PGM one) because of the deadlock risk.
539	* @thread Any.
540	*/
541	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
542	{
543	int rc = pgmLock(pVM);
544	AssertRCReturn(rc, rc);
545
546	/*
547	* Query the Physical TLB entry for the page (may fail).
548	*/
549	PPGMPAGEMAPTLBE pTlbe;
550	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
551	if (RT_SUCCESS(rc))
552	{
553	PPGMPAGE pPage = pTlbe->pPage;
554	#if 1
555	/* MMIO pages doesn't have any readable backing. */
556	if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
557	rc = VERR_PGM_PHYS_PAGE_RESERVED;
558	#else
559	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
560	rc = VERR_PGM_PHYS_PAGE_RESERVED;
561	#endif
562	else
563	{
564	/*
565	* Now, just perform the locking and calculate the return address.
566	*/
567	PPGMPAGEMAP pMap = pTlbe->pMap;
568	if (pMap)
569	pMap->cRefs++;
570
571	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
572	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
573	{
574	if (cLocks == 0)
575	pVM->pgm.s.cReadLockedPages++;
576	PGM_PAGE_INC_READ_LOCKS(pPage);
577	}
578	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
579	{
580	PGM_PAGE_INC_READ_LOCKS(pPage);
581	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
582	if (pMap)
583	pMap->cRefs++; /* Extra ref to prevent it from going away. */
584	}
585
586	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
587	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
588	pLock->pvMap = pMap;
589	}
590	}
591
592	pgmUnlock(pVM);
593	return rc;
594	}
595
596
597	/**
598	* Requests the mapping of multiple guest page into ring-3, external threads.
599	*
600	* When you're done with the pages, call PGMPhysBulkReleasePageMappingLock()
601	* ASAP to release them.
602	*
603	* This API will assume your intention is to write to the pages, and will
604	* therefore replace shared and zero pages. If you do not intend to modify the
605	* pages, use the PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal() API.
606	*
607	* @returns VBox status code.
608	* @retval VINF_SUCCESS on success.
609	* @retval VERR_PGM_PHYS_PAGE_RESERVED if any of the pages has no physical
610	* backing or if any of the pages the page has any active access
611	* handlers. The caller must fall back on using PGMR3PhysWriteExternal.
612	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if @a paGCPhysPages contains
613	* an invalid physical address.
614	*
615	* @param pVM The cross context VM structure.
616	* @param cPages Number of pages to lock.
617	* @param paGCPhysPages The guest physical address of the pages that
618	* should be mapped (@a cPages entries).
619	* @param papvPages Where to store the ring-3 mapping addresses
620	* corresponding to @a paGCPhysPages.
621	* @param paLocks Where to store the locking information that
622	* pfnPhysBulkReleasePageMappingLock needs (@a cPages
623	* in length).
624	*
625	* @remark Avoid calling this API from within critical sections (other than the
626	* PGM one) because of the deadlock risk when we have to delegating the
627	* task to an EMT.
628	* @thread Any.
629	*/
630	VMMR3DECL(int) PGMR3PhysBulkGCPhys2CCPtrExternal(PVM pVM, uint32_t cPages, PCRTGCPHYS paGCPhysPages,
631	void **papvPages, PPGMPAGEMAPLOCK paLocks)
632	{
633	Assert(cPages > 0);
634	AssertPtr(papvPages);
635	AssertPtr(paLocks);
636
637	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
638
639	int rc = pgmLock(pVM);
640	AssertRCReturn(rc, rc);
641
642	/*
643	* Lock the pages one by one.
644	* The loop body is similar to PGMR3PhysGCPhys2CCPtrExternal.
645	*/
646	int32_t cNextYield = 128;
647	uint32_t iPage;
648	for (iPage = 0; iPage < cPages; iPage++)
649	{
650	if (--cNextYield > 0)
651	{ /* likely */ }
652	else
653	{
654	pgmUnlock(pVM);
655	ASMNopPause();
656	pgmLock(pVM);
657	cNextYield = 128;
658	}
659
660	/*
661	* Query the Physical TLB entry for the page (may fail).
662	*/
663	PPGMPAGEMAPTLBE pTlbe;
664	rc = pgmPhysPageQueryTlbe(pVM, paGCPhysPages[iPage], &pTlbe);
665	if (RT_SUCCESS(rc))
666	{ }
667	else
668	break;
669	PPGMPAGE pPage = pTlbe->pPage;
670
671	/*
672	* No MMIO or active access handlers.
673	*/
674	if ( !PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)
675	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
676	{ }
677	else
678	{
679	rc = VERR_PGM_PHYS_PAGE_RESERVED;
680	break;
681	}
682
683	/*
684	* The page must be in the allocated state and not be a dirty pool page.
685	* We can handle converting a write monitored page to an allocated one, but
686	* anything more complicated must be delegated to an EMT.
687	*/
688	bool fDelegateToEmt = false;
689	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
690	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
691	fDelegateToEmt = pgmPoolIsDirtyPage(pVM, paGCPhysPages[iPage]);
692	#else
693	fDelegateToEmt = false;
694	#endif
695	else if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
696	{
697	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
698	if (!pgmPoolIsDirtyPage(pVM, paGCPhysPages[iPage]))
699	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, paGCPhysPages[iPage]);
700	else
701	fDelegateToEmt = true;
702	#endif
703	}
704	else
705	fDelegateToEmt = true;
706	if (!fDelegateToEmt)
707	{ }
708	else
709	{
710	/* We could do this delegation in bulk, but considered too much work vs gain. */
711	pgmUnlock(pVM);
712	rc = VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
713	pVM, &paGCPhysPages[iPage], &papvPages[iPage], &paLocks[iPage]);
714	pgmLock(pVM);
715	if (RT_FAILURE(rc))
716	break;
717	cNextYield = 128;
718	}
719
720	/*
721	* Now, just perform the locking and address calculation.
722	*/
723	PPGMPAGEMAP pMap = pTlbe->pMap;
724	if (pMap)
725	pMap->cRefs++;
726
727	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
728	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
729	{
730	if (cLocks == 0)
731	pVM->pgm.s.cWriteLockedPages++;
732	PGM_PAGE_INC_WRITE_LOCKS(pPage);
733	}
734	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
735	{
736	PGM_PAGE_INC_WRITE_LOCKS(pPage);
737	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", paGCPhysPages[iPage], pPage));
738	if (pMap)
739	pMap->cRefs++; /* Extra ref to prevent it from going away. */
740	}
741
742	papvPages[iPage] = (void *)((uintptr_t)pTlbe->pv \| (uintptr_t)(paGCPhysPages[iPage] & PAGE_OFFSET_MASK));
743	paLocks[iPage].uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
744	paLocks[iPage].pvMap = pMap;
745	}
746
747	pgmUnlock(pVM);
748
749	/*
750	* On failure we must unlock any pages we managed to get already.
751	*/
752	if (RT_FAILURE(rc) && iPage > 0)
753	PGMPhysBulkReleasePageMappingLocks(pVM, iPage, paLocks);
754
755	return rc;
756	}
757
758
759	/**
760	* Requests the mapping of multiple guest page into ring-3, for reading only,
761	* external threads.
762	*
763	* When you're done with the pages, call PGMPhysReleasePageMappingLock() ASAP
764	* to release them.
765	*
766	* @returns VBox status code.
767	* @retval VINF_SUCCESS on success.
768	* @retval VERR_PGM_PHYS_PAGE_RESERVED if any of the pages has no physical
769	* backing or if any of the pages the page has an active ALL access
770	* handler. The caller must fall back on using PGMR3PhysWriteExternal.
771	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if @a paGCPhysPages contains
772	* an invalid physical address.
773	*
774	* @param pVM The cross context VM structure.
775	* @param cPages Number of pages to lock.
776	* @param paGCPhysPages The guest physical address of the pages that
777	* should be mapped (@a cPages entries).
778	* @param papvPages Where to store the ring-3 mapping addresses
779	* corresponding to @a paGCPhysPages.
780	* @param paLocks Where to store the lock information that
781	* pfnPhysReleasePageMappingLock needs (@a cPages
782	* in length).
783	*
784	* @remark Avoid calling this API from within critical sections (other than
785	* the PGM one) because of the deadlock risk.
786	* @thread Any.
787	*/
788	VMMR3DECL(int) PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal(PVM pVM, uint32_t cPages, PCRTGCPHYS paGCPhysPages,
789	void const **papvPages, PPGMPAGEMAPLOCK paLocks)
790	{
791	Assert(cPages > 0);
792	AssertPtr(papvPages);
793	AssertPtr(paLocks);
794
795	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
796
797	int rc = pgmLock(pVM);
798	AssertRCReturn(rc, rc);
799
800	/*
801	* Lock the pages one by one.
802	* The loop body is similar to PGMR3PhysGCPhys2CCPtrReadOnlyExternal.
803	*/
804	int32_t cNextYield = 256;
805	uint32_t iPage;
806	for (iPage = 0; iPage < cPages; iPage++)
807	{
808	if (--cNextYield > 0)
809	{ /* likely */ }
810	else
811	{
812	pgmUnlock(pVM);
813	ASMNopPause();
814	pgmLock(pVM);
815	cNextYield = 256;
816	}
817
818	/*
819	* Query the Physical TLB entry for the page (may fail).
820	*/
821	PPGMPAGEMAPTLBE pTlbe;
822	rc = pgmPhysPageQueryTlbe(pVM, paGCPhysPages[iPage], &pTlbe);
823	if (RT_SUCCESS(rc))
824	{ }
825	else
826	break;
827	PPGMPAGE pPage = pTlbe->pPage;
828
829	/*
830	* No MMIO or active all access handlers, everything else can be accessed.
831	*/
832	if ( !PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)
833	&& !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
834	{ }
835	else
836	{
837	rc = VERR_PGM_PHYS_PAGE_RESERVED;
838	break;
839	}
840
841	/*
842	* Now, just perform the locking and address calculation.
843	*/
844	PPGMPAGEMAP pMap = pTlbe->pMap;
845	if (pMap)
846	pMap->cRefs++;
847
848	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
849	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
850	{
851	if (cLocks == 0)
852	pVM->pgm.s.cReadLockedPages++;
853	PGM_PAGE_INC_READ_LOCKS(pPage);
854	}
855	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
856	{
857	PGM_PAGE_INC_READ_LOCKS(pPage);
858	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", paGCPhysPages[iPage], pPage));
859	if (pMap)
860	pMap->cRefs++; /* Extra ref to prevent it from going away. */
861	}
862
863	papvPages[iPage] = (void *)((uintptr_t)pTlbe->pv \| (uintptr_t)(paGCPhysPages[iPage] & PAGE_OFFSET_MASK));
864	paLocks[iPage].uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
865	paLocks[iPage].pvMap = pMap;
866	}
867
868	pgmUnlock(pVM);
869
870	/*
871	* On failure we must unlock any pages we managed to get already.
872	*/
873	if (RT_FAILURE(rc) && iPage > 0)
874	PGMPhysBulkReleasePageMappingLocks(pVM, iPage, paLocks);
875
876	return rc;
877	}
878
879
880	#define MAKE_LEAF(a_pNode) \
881	do { \
882	(a_pNode)->pLeftR3 = NIL_RTR3PTR; \
883	(a_pNode)->pRightR3 = NIL_RTR3PTR; \
884	(a_pNode)->pLeftR0 = NIL_RTR0PTR; \
885	(a_pNode)->pRightR0 = NIL_RTR0PTR; \
886	} while (0)
887
888	#define INSERT_LEFT(a_pParent, a_pNode) \
889	do { \
890	(a_pParent)->pLeftR3 = (a_pNode); \
891	(a_pParent)->pLeftR0 = (a_pNode)->pSelfR0; \
892	} while (0)
893	#define INSERT_RIGHT(a_pParent, a_pNode) \
894	do { \
895	(a_pParent)->pRightR3 = (a_pNode); \
896	(a_pParent)->pRightR0 = (a_pNode)->pSelfR0; \
897	} while (0)
898
899
900	/**
901	* Recursive tree builder.
902	*
903	* @param ppRam Pointer to the iterator variable.
904	* @param iDepth The current depth. Inserts a leaf node if 0.
905	*/
906	static PPGMRAMRANGE pgmR3PhysRebuildRamRangeSearchTreesRecursively(PPGMRAMRANGE *ppRam, int iDepth)
907	{
908	PPGMRAMRANGE pRam;
909	if (iDepth <= 0)
910	{
911	/*
912	* Leaf node.
913	*/
914	pRam = *ppRam;
915	if (pRam)
916	{
917	*ppRam = pRam->pNextR3;
918	MAKE_LEAF(pRam);
919	}
920	}
921	else
922	{
923
924	/*
925	* Intermediate node.
926	*/
927	PPGMRAMRANGE pLeft = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
928
929	pRam = *ppRam;
930	if (!pRam)
931	return pLeft;
932	*ppRam = pRam->pNextR3;
933	MAKE_LEAF(pRam);
934	INSERT_LEFT(pRam, pLeft);
935
936	PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
937	if (pRight)
938	INSERT_RIGHT(pRam, pRight);
939	}
940	return pRam;
941	}
942
943
944	/**
945	* Rebuilds the RAM range search trees.
946	*
947	* @param pVM The cross context VM structure.
948	*/
949	static void pgmR3PhysRebuildRamRangeSearchTrees(PVM pVM)
950	{
951
952	/*
953	* Create the reasonably balanced tree in a sequential fashion.
954	* For simplicity (laziness) we use standard recursion here.
955	*/
956	int iDepth = 0;
957	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
958	PPGMRAMRANGE pRoot = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, 0);
959	while (pRam)
960	{
961	PPGMRAMRANGE pLeft = pRoot;
962
963	pRoot = pRam;
964	pRam = pRam->pNextR3;
965	MAKE_LEAF(pRoot);
966	INSERT_LEFT(pRoot, pLeft);
967
968	PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, iDepth);
969	if (pRight)
970	INSERT_RIGHT(pRoot, pRight);
971	/** @todo else: rotate the tree. */
972
973	iDepth++;
974	}
975
976	pVM->pgm.s.pRamRangeTreeR3 = pRoot;
977	pVM->pgm.s.pRamRangeTreeR0 = pRoot ? pRoot->pSelfR0 : NIL_RTR0PTR;
978
979	#ifdef VBOX_STRICT
980	/*
981	* Verify that the above code works.
982	*/
983	unsigned cRanges = 0;
984	for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
985	cRanges++;
986	Assert(cRanges > 0);
987
988	unsigned cMaxDepth = ASMBitLastSetU32(cRanges);
989	if ((1U << cMaxDepth) < cRanges)
990	cMaxDepth++;
991
992	for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
993	{
994	unsigned cDepth = 0;
995	PPGMRAMRANGE pRam2 = pVM->pgm.s.pRamRangeTreeR3;
996	for (;;)
997	{
998	if (pRam == pRam2)
999	break;
1000	Assert(pRam2);
1001	if (pRam->GCPhys < pRam2->GCPhys)
1002	pRam2 = pRam2->pLeftR3;
1003	else
1004	pRam2 = pRam2->pRightR3;
1005	}
1006	AssertMsg(cDepth <= cMaxDepth, ("cDepth=%d cMaxDepth=%d\n", cDepth, cMaxDepth));
1007	}
1008	#endif /* VBOX_STRICT */
1009	}
1010
1011	#undef MAKE_LEAF
1012	#undef INSERT_LEFT
1013	#undef INSERT_RIGHT
1014
1015	/**
1016	* Relinks the RAM ranges using the pSelfRC and pSelfR0 pointers.
1017	*
1018	* Called when anything was relocated.
1019	*
1020	* @param pVM The cross context VM structure.
1021	*/
1022	void pgmR3PhysRelinkRamRanges(PVM pVM)
1023	{
1024	PPGMRAMRANGE pCur;
1025
1026	#ifdef VBOX_STRICT
1027	for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
1028	{
1029	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfR0 == MMHyperCCToR0(pVM, pCur));
1030	Assert((pCur->GCPhys & PAGE_OFFSET_MASK) == 0);
1031	Assert((pCur->GCPhysLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
1032	Assert((pCur->cb & PAGE_OFFSET_MASK) == 0);
1033	Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
1034	for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesXR3; pCur2; pCur2 = pCur2->pNextR3)
1035	Assert( pCur2 == pCur
1036	\|\| strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
1037	}
1038	#endif
1039
1040	pCur = pVM->pgm.s.pRamRangesXR3;
1041	if (pCur)
1042	{
1043	pVM->pgm.s.pRamRangesXR0 = pCur->pSelfR0;
1044
1045	for (; pCur->pNextR3; pCur = pCur->pNextR3)
1046	pCur->pNextR0 = pCur->pNextR3->pSelfR0;
1047
1048	Assert(pCur->pNextR0 == NIL_RTR0PTR);
1049	}
1050	else
1051	{
1052	Assert(pVM->pgm.s.pRamRangesXR0 == NIL_RTR0PTR);
1053	}
1054	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
1055
1056	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
1057	}
1058
1059
1060	/**
1061	* Links a new RAM range into the list.
1062	*
1063	* @param pVM The cross context VM structure.
1064	* @param pNew Pointer to the new list entry.
1065	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
1066	*/
1067	static void pgmR3PhysLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, PPGMRAMRANGE pPrev)
1068	{
1069	AssertMsg(pNew->pszDesc, ("%RGp-%RGp\n", pNew->GCPhys, pNew->GCPhysLast));
1070	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfR0 == MMHyperCCToR0(pVM, pNew));
1071
1072	pgmLock(pVM);
1073
1074	PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesXR3;
1075	pNew->pNextR3 = pRam;
1076	pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
1077
1078	if (pPrev)
1079	{
1080	pPrev->pNextR3 = pNew;
1081	pPrev->pNextR0 = pNew->pSelfR0;
1082	}
1083	else
1084	{
1085	pVM->pgm.s.pRamRangesXR3 = pNew;
1086	pVM->pgm.s.pRamRangesXR0 = pNew->pSelfR0;
1087	}
1088	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
1089
1090	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
1091	pgmUnlock(pVM);
1092	}
1093
1094
1095	/**
1096	* Unlink an existing RAM range from the list.
1097	*
1098	* @param pVM The cross context VM structure.
1099	* @param pRam Pointer to the new list entry.
1100	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
1101	*/
1102	static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
1103	{
1104	Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesXR3 == pRam);
1105	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfR0 == MMHyperCCToR0(pVM, pRam));
1106
1107	pgmLock(pVM);
1108
1109	PPGMRAMRANGE pNext = pRam->pNextR3;
1110	if (pPrev)
1111	{
1112	pPrev->pNextR3 = pNext;
1113	pPrev->pNextR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
1114	}
1115	else
1116	{
1117	Assert(pVM->pgm.s.pRamRangesXR3 == pRam);
1118	pVM->pgm.s.pRamRangesXR3 = pNext;
1119	pVM->pgm.s.pRamRangesXR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
1120	}
1121	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
1122
1123	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
1124	pgmUnlock(pVM);
1125	}
1126
1127
1128	/**
1129	* Unlink an existing RAM range from the list.
1130	*
1131	* @param pVM The cross context VM structure.
1132	* @param pRam Pointer to the new list entry.
1133	*/
1134	static void pgmR3PhysUnlinkRamRange(PVM pVM, PPGMRAMRANGE pRam)
1135	{
1136	pgmLock(pVM);
1137
1138	/* find prev. */
1139	PPGMRAMRANGE pPrev = NULL;
1140	PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesXR3;
1141	while (pCur != pRam)
1142	{
1143	pPrev = pCur;
1144	pCur = pCur->pNextR3;
1145	}
1146	AssertFatal(pCur);
1147
1148	pgmR3PhysUnlinkRamRange2(pVM, pRam, pPrev);
1149	pgmUnlock(pVM);
1150	}
1151
1152
1153	/**
1154	* Frees a range of pages, replacing them with ZERO pages of the specified type.
1155	*
1156	* @returns VBox status code.
1157	* @param pVM The cross context VM structure.
1158	* @param pRam The RAM range in which the pages resides.
1159	* @param GCPhys The address of the first page.
1160	* @param GCPhysLast The address of the last page.
1161	* @param enmType The page type to replace then with.
1162	*/
1163	static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, PGMPAGETYPE enmType)
1164	{
1165	PGM_LOCK_ASSERT_OWNER(pVM);
1166	uint32_t cPendingPages = 0;
1167	PGMMFREEPAGESREQ pReq;
1168	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1169	AssertLogRelRCReturn(rc, rc);
1170
1171	/* Iterate the pages. */
1172	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
1173	uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> PAGE_SHIFT) + 1;
1174	while (cPagesLeft-- > 0)
1175	{
1176	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys, enmType);
1177	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
1178
1179	PGM_PAGE_SET_TYPE(pVM, pPageDst, enmType);
1180
1181	GCPhys += PAGE_SIZE;
1182	pPageDst++;
1183	}
1184
1185	if (cPendingPages)
1186	{
1187	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
1188	AssertLogRelRCReturn(rc, rc);
1189	}
1190	GMMR3FreePagesCleanup(pReq);
1191
1192	return rc;
1193	}
1194
1195	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
1196
1197	/**
1198	* Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
1199	*
1200	* This is only called on one of the EMTs while the other ones are waiting for
1201	* it to complete this function.
1202	*
1203	* @returns VINF_SUCCESS (VBox strict status code).
1204	* @param pVM The cross context VM structure.
1205	* @param pVCpu The cross context virtual CPU structure of the calling EMT. Unused.
1206	* @param pvUser User parameter
1207	*/
1208	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1209	{
1210	uintptr_t paUser = (uintptr_t )pvUser;
1211	bool fInflate = !!paUser[0];
1212	unsigned cPages = paUser[1];
1213	RTGCPHYS paPhysPage = (RTGCPHYS )paUser[2];
1214	uint32_t cPendingPages = 0;
1215	PGMMFREEPAGESREQ pReq;
1216	int rc;
1217
1218	Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
1219	pgmLock(pVM);
1220
1221	if (fInflate)
1222	{
1223	/* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
1224	pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
1225
1226	/* Replace pages with ZERO pages. */
1227	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1228	if (RT_FAILURE(rc))
1229	{
1230	pgmUnlock(pVM);
1231	AssertLogRelRC(rc);
1232	return rc;
1233	}
1234
1235	/* Iterate the pages. */
1236	for (unsigned i = 0; i < cPages; i++)
1237	{
1238	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
1239	if ( pPage == NULL
1240	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM)
1241	{
1242	Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], pPage ? PGM_PAGE_GET_TYPE(pPage) : 0));
1243	break;
1244	}
1245
1246	LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
1247
1248	/* Flush the shadow PT if this page was previously used as a guest page table. */
1249	pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
1250
1251	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i], (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage));
1252	if (RT_FAILURE(rc))
1253	{
1254	pgmUnlock(pVM);
1255	AssertLogRelRC(rc);
1256	return rc;
1257	}
1258	Assert(PGM_PAGE_IS_ZERO(pPage));
1259	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_BALLOONED);
1260	}
1261
1262	if (cPendingPages)
1263	{
1264	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
1265	if (RT_FAILURE(rc))
1266	{
1267	pgmUnlock(pVM);
1268	AssertLogRelRC(rc);
1269	return rc;
1270	}
1271	}
1272	GMMR3FreePagesCleanup(pReq);
1273	}
1274	else
1275	{
1276	/* Iterate the pages. */
1277	for (unsigned i = 0; i < cPages; i++)
1278	{
1279	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
1280	AssertBreak(pPage && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM);
1281
1282	LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
1283
1284	Assert(PGM_PAGE_IS_BALLOONED(pPage));
1285
1286	/* Change back to zero page. (NEM does not need to be informed.) */
1287	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1288	}
1289
1290	/* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
1291	}
1292
1293	/* Notify GMM about the balloon change. */
1294	rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
1295	if (RT_SUCCESS(rc))
1296	{
1297	if (!fInflate)
1298	{
1299	Assert(pVM->pgm.s.cBalloonedPages >= cPages);
1300	pVM->pgm.s.cBalloonedPages -= cPages;
1301	}
1302	else
1303	pVM->pgm.s.cBalloonedPages += cPages;
1304	}
1305
1306	pgmUnlock(pVM);
1307
1308	/* Flush the recompiler's TLB as well. */
1309	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1310	CPUMSetChangedFlags(&pVM->aCpus[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1311
1312	AssertLogRelRC(rc);
1313	return rc;
1314	}
1315
1316
1317	/**
1318	* Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
1319	*
1320	* @returns VBox status code.
1321	* @param pVM The cross context VM structure.
1322	* @param fInflate Inflate or deflate memory balloon
1323	* @param cPages Number of pages to free
1324	* @param paPhysPage Array of guest physical addresses
1325	*/
1326	static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1327	{
1328	uintptr_t paUser[3];
1329
1330	paUser[0] = fInflate;
1331	paUser[1] = cPages;
1332	paUser[2] = (uintptr_t)paPhysPage;
1333	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1334	AssertRC(rc);
1335
1336	/* Made a copy in PGMR3PhysFreeRamPages; free it here. */
1337	RTMemFree(paPhysPage);
1338	}
1339
1340	#endif /* 64-bit host && (Windows \|\| Solaris \|\| Linux \|\| FreeBSD) */
1341
1342	/**
1343	* Inflate or deflate a memory balloon
1344	*
1345	* @returns VBox status code.
1346	* @param pVM The cross context VM structure.
1347	* @param fInflate Inflate or deflate memory balloon
1348	* @param cPages Number of pages to free
1349	* @param paPhysPage Array of guest physical addresses
1350	*/
1351	VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1352	{
1353	/* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
1354	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
1355	int rc;
1356
1357	/* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
1358	AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
1359
1360	/* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
1361	* In the SMP case we post a request packet to postpone the job.
1362	*/
1363	if (pVM->cCpus > 1)
1364	{
1365	unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
1366	RTGCPHYS paPhysPageCopy = (RTGCPHYS )RTMemAlloc(cbPhysPage);
1367	AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
1368
1369	memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
1370
1371	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4, pVM, fInflate, cPages, paPhysPageCopy);
1372	AssertRC(rc);
1373	}
1374	else
1375	{
1376	uintptr_t paUser[3];
1377
1378	paUser[0] = fInflate;
1379	paUser[1] = cPages;
1380	paUser[2] = (uintptr_t)paPhysPage;
1381	rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1382	AssertRC(rc);
1383	}
1384	return rc;
1385
1386	#else
1387	NOREF(pVM); NOREF(fInflate); NOREF(cPages); NOREF(paPhysPage);
1388	return VERR_NOT_IMPLEMENTED;
1389	#endif
1390	}
1391
1392
1393	/**
1394	* Rendezvous callback used by PGMR3WriteProtectRAM that write protects all
1395	* physical RAM.
1396	*
1397	* This is only called on one of the EMTs while the other ones are waiting for
1398	* it to complete this function.
1399	*
1400	* @returns VINF_SUCCESS (VBox strict status code).
1401	* @param pVM The cross context VM structure.
1402	* @param pVCpu The cross context virtual CPU structure of the calling EMT. Unused.
1403	* @param pvUser User parameter, unused.
1404	*/
1405	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1406	{
1407	int rc = VINF_SUCCESS;
1408	NOREF(pvUser); NOREF(pVCpu);
1409
1410	pgmLock(pVM);
1411	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1412	pgmPoolResetDirtyPages(pVM);
1413	#endif
1414
1415	/** @todo pointless to write protect the physical page pointed to by RSP. */
1416
1417	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1418	pRam;
1419	pRam = pRam->CTX_SUFF(pNext))
1420	{
1421	uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1422	for (uint32_t iPage = 0; iPage < cPages; iPage++)
1423	{
1424	PPGMPAGE pPage = &pRam->aPages[iPage];
1425	PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1426
1427	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1428	\|\| enmPageType == PGMPAGETYPE_MMIO2)
1429	{
1430	/*
1431	* A RAM page.
1432	*/
1433	switch (PGM_PAGE_GET_STATE(pPage))
1434	{
1435	case PGM_PAGE_STATE_ALLOCATED:
1436	/** @todo Optimize this: Don't always re-enable write
1437	* monitoring if the page is known to be very busy. */
1438	if (PGM_PAGE_IS_WRITTEN_TO(pPage))
1439	PGM_PAGE_CLEAR_WRITTEN_TO(pVM, pPage);
1440
1441	pgmPhysPageWriteMonitor(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1442	break;
1443
1444	case PGM_PAGE_STATE_SHARED:
1445	AssertFailed();
1446	break;
1447
1448	case PGM_PAGE_STATE_WRITE_MONITORED: /* nothing to change. */
1449	default:
1450	break;
1451	}
1452	}
1453	}
1454	}
1455	pgmR3PoolWriteProtectPages(pVM);
1456	PGM_INVL_ALL_VCPU_TLBS(pVM);
1457	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1458	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1459
1460	pgmUnlock(pVM);
1461	return rc;
1462	}
1463
1464	/**
1465	* Protect all physical RAM to monitor writes
1466	*
1467	* @returns VBox status code.
1468	* @param pVM The cross context VM structure.
1469	*/
1470	VMMR3DECL(int) PGMR3PhysWriteProtectRAM(PVM pVM)
1471	{
1472	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1473
1474	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysWriteProtectRAMRendezvous, NULL);
1475	AssertRC(rc);
1476	return rc;
1477	}
1478
1479
1480	/**
1481	* Gets the number of ram ranges.
1482	*
1483	* @returns Number of ram ranges. Returns UINT32_MAX if @a pVM is invalid.
1484	* @param pVM The cross context VM structure.
1485	*/
1486	VMMR3DECL(uint32_t) PGMR3PhysGetRamRangeCount(PVM pVM)
1487	{
1488	VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
1489
1490	pgmLock(pVM);
1491	uint32_t cRamRanges = 0;
1492	for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext))
1493	cRamRanges++;
1494	pgmUnlock(pVM);
1495	return cRamRanges;
1496	}
1497
1498
1499	/**
1500	* Get information about a range.
1501	*
1502	* @returns VINF_SUCCESS or VERR_OUT_OF_RANGE.
1503	* @param pVM The cross context VM structure.
1504	* @param iRange The ordinal of the range.
1505	* @param pGCPhysStart Where to return the start of the range. Optional.
1506	* @param pGCPhysLast Where to return the address of the last byte in the
1507	* range. Optional.
1508	* @param ppszDesc Where to return the range description. Optional.
1509	* @param pfIsMmio Where to indicate that this is a pure MMIO range.
1510	* Optional.
1511	*/
1512	VMMR3DECL(int) PGMR3PhysGetRange(PVM pVM, uint32_t iRange, PRTGCPHYS pGCPhysStart, PRTGCPHYS pGCPhysLast,
1513	const char *ppszDesc, bool pfIsMmio)
1514	{
1515	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1516
1517	pgmLock(pVM);
1518	uint32_t iCurRange = 0;
1519	for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext), iCurRange++)
1520	if (iCurRange == iRange)
1521	{
1522	if (pGCPhysStart)
1523	*pGCPhysStart = pCur->GCPhys;
1524	if (pGCPhysLast)
1525	*pGCPhysLast = pCur->GCPhysLast;
1526	if (ppszDesc)
1527	*ppszDesc = pCur->pszDesc;
1528	if (pfIsMmio)
1529	*pfIsMmio = !!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO);
1530
1531	pgmUnlock(pVM);
1532	return VINF_SUCCESS;
1533	}
1534	pgmUnlock(pVM);
1535	return VERR_OUT_OF_RANGE;
1536	}
1537
1538
1539	/**
1540	* Query the amount of free memory inside VMMR0
1541	*
1542	* @returns VBox status code.
1543	* @param pUVM The user mode VM handle.
1544	* @param pcbAllocMem Where to return the amount of memory allocated
1545	* by VMs.
1546	* @param pcbFreeMem Where to return the amount of memory that is
1547	* allocated from the host but not currently used
1548	* by any VMs.
1549	* @param pcbBallonedMem Where to return the sum of memory that is
1550	* currently ballooned by the VMs.
1551	* @param pcbSharedMem Where to return the amount of memory that is
1552	* currently shared.
1553	*/
1554	VMMR3DECL(int) PGMR3QueryGlobalMemoryStats(PUVM pUVM, uint64_t pcbAllocMem, uint64_t pcbFreeMem,
1555	uint64_t pcbBallonedMem, uint64_t pcbSharedMem)
1556	{
1557	UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
1558	VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
1559
1560	uint64_t cAllocPages = 0;
1561	uint64_t cFreePages = 0;
1562	uint64_t cBalloonPages = 0;
1563	uint64_t cSharedPages = 0;
1564	int rc = GMMR3QueryHypervisorMemoryStats(pUVM->pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
1565	AssertRCReturn(rc, rc);
1566
1567	if (pcbAllocMem)
1568	pcbAllocMem = cAllocPages _4K;
1569
1570	if (pcbFreeMem)
1571	pcbFreeMem = cFreePages _4K;
1572
1573	if (pcbBallonedMem)
1574	pcbBallonedMem = cBalloonPages _4K;
1575
1576	if (pcbSharedMem)
1577	pcbSharedMem = cSharedPages _4K;
1578
1579	Log(("PGMR3QueryVMMMemoryStats: all=%llx free=%llx ballooned=%llx shared=%llx\n",
1580	cAllocPages, cFreePages, cBalloonPages, cSharedPages));
1581	return VINF_SUCCESS;
1582	}
1583
1584
1585	/**
1586	* Query memory stats for the VM.
1587	*
1588	* @returns VBox status code.
1589	* @param pUVM The user mode VM handle.
1590	* @param pcbTotalMem Where to return total amount memory the VM may
1591	* possibly use.
1592	* @param pcbPrivateMem Where to return the amount of private memory
1593	* currently allocated.
1594	* @param pcbSharedMem Where to return the amount of actually shared
1595	* memory currently used by the VM.
1596	* @param pcbZeroMem Where to return the amount of memory backed by
1597	* zero pages.
1598	*
1599	* @remarks The total mem is normally larger than the sum of the three
1600	* components. There are two reasons for this, first the amount of
1601	* shared memory is what we're sure is shared instead of what could
1602	* possibly be shared with someone. Secondly, because the total may
1603	* include some pure MMIO pages that doesn't go into any of the three
1604	* sub-counts.
1605	*
1606	* @todo Why do we return reused shared pages instead of anything that could
1607	* potentially be shared? Doesn't this mean the first VM gets a much
1608	* lower number of shared pages?
1609	*/
1610	VMMR3DECL(int) PGMR3QueryMemoryStats(PUVM pUVM, uint64_t pcbTotalMem, uint64_t pcbPrivateMem,
1611	uint64_t pcbSharedMem, uint64_t pcbZeroMem)
1612	{
1613	UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
1614	PVM pVM = pUVM->pVM;
1615	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1616
1617	if (pcbTotalMem)
1618	pcbTotalMem = (uint64_t)pVM->pgm.s.cAllPages PAGE_SIZE;
1619
1620	if (pcbPrivateMem)
1621	pcbPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages PAGE_SIZE;
1622
1623	if (pcbSharedMem)
1624	pcbSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages PAGE_SIZE;
1625
1626	if (pcbZeroMem)
1627	pcbZeroMem = (uint64_t)pVM->pgm.s.cZeroPages PAGE_SIZE;
1628
1629	Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
1630	return VINF_SUCCESS;
1631	}
1632
1633
1634	/**
1635	* PGMR3PhysRegisterRam worker that initializes and links a RAM range.
1636	*
1637	* @param pVM The cross context VM structure.
1638	* @param pNew The new RAM range.
1639	* @param GCPhys The address of the RAM range.
1640	* @param GCPhysLast The last address of the RAM range.
1641	* @param RCPtrNew The RC address if the range is floating. NIL_RTRCPTR
1642	* if in HMA.
1643	* @param R0PtrNew Ditto for R0.
1644	* @param pszDesc The description.
1645	* @param pPrev The previous RAM range (for linking).
1646	*/
1647	static void pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
1648	RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
1649	{
1650	/*
1651	* Initialize the range.
1652	*/
1653	pNew->pSelfR0 = R0PtrNew != NIL_RTR0PTR ? R0PtrNew : MMHyperCCToR0(pVM, pNew);
1654	pNew->GCPhys = GCPhys;
1655	pNew->GCPhysLast = GCPhysLast;
1656	pNew->cb = GCPhysLast - GCPhys + 1;
1657	pNew->pszDesc = pszDesc;
1658	pNew->fFlags = RCPtrNew != NIL_RTRCPTR ? PGM_RAM_RANGE_FLAGS_FLOATING : 0;
1659	pNew->pvR3 = NULL;
1660	pNew->paLSPages = NULL;
1661
1662	uint32_t const cPages = pNew->cb >> PAGE_SHIFT;
1663	RTGCPHYS iPage = cPages;
1664	while (iPage-- > 0)
1665	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
1666
1667	/* Update the page count stats. */
1668	pVM->pgm.s.cZeroPages += cPages;
1669	pVM->pgm.s.cAllPages += cPages;
1670
1671	/*
1672	* Link it.
1673	*/
1674	pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
1675	}
1676
1677
1678	#ifndef PGM_WITHOUT_MAPPINGS
1679	/**
1680	* @callback_method_impl{FNPGMRELOCATE, Relocate a floating RAM range.}
1681	* @sa pgmR3PhysMMIO2ExRangeRelocate
1682	*/
1683	static DECLCALLBACK(bool) pgmR3PhysRamRangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew,
1684	PGMRELOCATECALL enmMode, void *pvUser)
1685	{
1686	PPGMRAMRANGE pRam = (PPGMRAMRANGE)pvUser;
1687	Assert(pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
1688	Assert(pRam->pSelfRC == GCPtrOld + PAGE_SIZE); RT_NOREF_PV(GCPtrOld);
1689
1690	switch (enmMode)
1691	{
1692	case PGMRELOCATECALL_SUGGEST:
1693	return true;
1694
1695	case PGMRELOCATECALL_RELOCATE:
1696	{
1697	/*
1698	* Update myself, then relink all the ranges and flush the RC TLB.
1699	*/
1700	pgmLock(pVM);
1701
1702	pRam->pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE);
1703
1704	pgmR3PhysRelinkRamRanges(pVM);
1705	for (unsigned i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
1706	pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
1707
1708	pgmUnlock(pVM);
1709	return true;
1710	}
1711
1712	default:
1713	AssertFailedReturn(false);
1714	}
1715	}
1716	#endif /* !PGM_WITHOUT_MAPPINGS */
1717
1718
1719	/**
1720	* PGMR3PhysRegisterRam worker that registers a high chunk.
1721	*
1722	* @returns VBox status code.
1723	* @param pVM The cross context VM structure.
1724	* @param GCPhys The address of the RAM.
1725	* @param cRamPages The number of RAM pages to register.
1726	* @param cbChunk The size of the PGMRAMRANGE guest mapping.
1727	* @param iChunk The chunk number.
1728	* @param pszDesc The RAM range description.
1729	* @param ppPrev Previous RAM range pointer. In/Out.
1730	*/
1731	static int pgmR3PhysRegisterHighRamChunk(PVM pVM, RTGCPHYS GCPhys, uint32_t cRamPages,
1732	uint32_t cbChunk, uint32_t iChunk, const char *pszDesc,
1733	PPGMRAMRANGE *ppPrev)
1734	{
1735	const char *pszDescChunk = iChunk == 0
1736	? pszDesc
1737	: MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, iChunk + 1);
1738	AssertReturn(pszDescChunk, VERR_NO_MEMORY);
1739
1740	/*
1741	* Allocate memory for the new chunk.
1742	*/
1743	size_t const cChunkPages = RT_ALIGN_Z(RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cRamPages]), PAGE_SIZE) >> PAGE_SHIFT;
1744	PSUPPAGE paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
1745	AssertReturn(paChunkPages, VERR_NO_TMP_MEMORY);
1746	RTR0PTR R0PtrChunk = NIL_RTR0PTR;
1747	void *pvChunk = NULL;
1748	int rc = SUPR3PageAllocEx(cChunkPages, 0 /fFlags/, &pvChunk, &R0PtrChunk, paChunkPages);
1749	if (RT_SUCCESS(rc))
1750	{
1751	Assert(R0PtrChunk != NIL_RTR0PTR);
1752	memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
1753
1754	PPGMRAMRANGE pNew = (PPGMRAMRANGE)pvChunk;
1755
1756	/*
1757	* Create a mapping and map the pages into it.
1758	* We push these in below the HMA.
1759	*/
1760	RTGCPTR GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
1761	#ifndef PGM_WITHOUT_MAPPINGS
1762	rc = PGMR3MapPT(pVM, GCPtrChunkMap, cbChunk, 0 /fFlags/, pgmR3PhysRamRangeRelocate, pNew, pszDescChunk);
1763	if (RT_SUCCESS(rc))
1764	#endif /* !PGM_WITHOUT_MAPPINGS */
1765	{
1766	pVM->pgm.s.GCPtrPrevRamRangeMapping = GCPtrChunkMap;
1767
1768	RTGCPTR const GCPtrChunk = GCPtrChunkMap + PAGE_SIZE;
1769	#ifndef PGM_WITHOUT_MAPPINGS
1770	RTGCPTR GCPtrPage = GCPtrChunk;
1771	for (uint32_t iPage = 0; iPage < cChunkPages && RT_SUCCESS(rc); iPage++, GCPtrPage += PAGE_SIZE)
1772	rc = PGMMap(pVM, GCPtrPage, paChunkPages[iPage].Phys, PAGE_SIZE, 0);
1773	if (RT_SUCCESS(rc))
1774	#endif /* !PGM_WITHOUT_MAPPINGS */
1775	{
1776	/*
1777	* Ok, init and link the range.
1778	*/
1779	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
1780	(RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
1781	*ppPrev = pNew;
1782	}
1783	}
1784
1785	if (RT_FAILURE(rc))
1786	SUPR3PageFreeEx(pvChunk, cChunkPages);
1787	}
1788
1789	RTMemTmpFree(paChunkPages);
1790	return rc;
1791	}
1792
1793
1794	/**
1795	* Sets up a range RAM.
1796	*
1797	* This will check for conflicting registrations, make a resource
1798	* reservation for the memory (with GMM), and setup the per-page
1799	* tracking structures (PGMPAGE).
1800	*
1801	* @returns VBox status code.
1802	* @param pVM The cross context VM structure.
1803	* @param GCPhys The physical address of the RAM.
1804	* @param cb The size of the RAM.
1805	* @param pszDesc The description - not copied, so, don't free or change it.
1806	*/
1807	VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
1808	{
1809	/*
1810	* Validate input.
1811	*/
1812	Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
1813	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
1814	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
1815	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
1816	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1817	AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
1818	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1819	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1820
1821	pgmLock(pVM);
1822
1823	/*
1824	* Find range location and check for conflicts.
1825	* (We don't lock here because the locking by EMT is only required on update.)
1826	*/
1827	PPGMRAMRANGE pPrev = NULL;
1828	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
1829	while (pRam && GCPhysLast >= pRam->GCPhys)
1830	{
1831	if ( GCPhysLast >= pRam->GCPhys
1832	&& GCPhys <= pRam->GCPhysLast)
1833	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
1834	GCPhys, GCPhysLast, pszDesc,
1835	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
1836	VERR_PGM_RAM_CONFLICT);
1837
1838	/* next */
1839	pPrev = pRam;
1840	pRam = pRam->pNextR3;
1841	}
1842
1843	/*
1844	* Register it with GMM (the API bitches).
1845	*/
1846	const RTGCPHYS cPages = cb >> PAGE_SHIFT;
1847	int rc = MMR3IncreaseBaseReservation(pVM, cPages);
1848	if (RT_FAILURE(rc))
1849	{
1850	pgmUnlock(pVM);
1851	return rc;
1852	}
1853
1854	if ( GCPhys >= _4G
1855	&& cPages > 256)
1856	{
1857	/*
1858	* The PGMRAMRANGE structures for the high memory can get very big.
1859	* In order to avoid SUPR3PageAllocEx allocation failures due to the
1860	* allocation size limit there and also to avoid being unable to find
1861	* guest mapping space for them, we split this memory up into 4MB in
1862	* (potential) raw-mode configs and 16MB chunks in forced AMD-V/VT-x
1863	* mode.
1864	*
1865	* The first and last page of each mapping are guard pages and marked
1866	* not-present. So, we've got 4186112 and 16769024 bytes available for
1867	* the PGMRAMRANGE structure.
1868	*
1869	* Note! The sizes used here will influence the saved state.
1870	*/
1871	uint32_t cbChunk = 16U*_1M;
1872	uint32_t cPagesPerChunk = 1048048; /* max ~1048059 */
1873	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 1048048 < 16U_1M - PAGE_SIZE 2);
1874	AssertRelease(RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
1875
1876	RTGCPHYS cPagesLeft = cPages;
1877	RTGCPHYS GCPhysChunk = GCPhys;
1878	uint32_t iChunk = 0;
1879	while (cPagesLeft > 0)
1880	{
1881	uint32_t cPagesInChunk = cPagesLeft;
1882	if (cPagesInChunk > cPagesPerChunk)
1883	cPagesInChunk = cPagesPerChunk;
1884
1885	rc = pgmR3PhysRegisterHighRamChunk(pVM, GCPhysChunk, cPagesInChunk, cbChunk, iChunk, pszDesc, &pPrev);
1886	AssertRCReturn(rc, rc);
1887
1888	/* advance */
1889	GCPhysChunk += (RTGCPHYS)cPagesInChunk << PAGE_SHIFT;
1890	cPagesLeft -= cPagesInChunk;
1891	iChunk++;
1892	}
1893	}
1894	else
1895	{
1896	/*
1897	* Allocate, initialize and link the new RAM range.
1898	*/
1899	const size_t cbRamRange = RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]);
1900	PPGMRAMRANGE pNew;
1901	rc = MMR3HyperAllocOnceNoRel(pVM, cbRamRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
1902	AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
1903
1904	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, NIL_RTR0PTR, pszDesc, pPrev);
1905	}
1906	pgmPhysInvalidatePageMapTLB(pVM);
1907
1908	/*
1909	* Notify NEM while holding the lock (experimental) and REM without (like always).
1910	*/
1911	rc = NEMR3NotifyPhysRamRegister(pVM, GCPhys, cb);
1912	pgmUnlock(pVM);
1913	#ifdef VBOX_WITH_REM
1914	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_RAM);
1915	#endif
1916	return rc;
1917	}
1918
1919
1920	/**
1921	* Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
1922	*
1923	* We do this late in the init process so that all the ROM and MMIO ranges have
1924	* been registered already and we don't go wasting memory on them.
1925	*
1926	* @returns VBox status code.
1927	*
1928	* @param pVM The cross context VM structure.
1929	*/
1930	int pgmR3PhysRamPreAllocate(PVM pVM)
1931	{
1932	Assert(pVM->pgm.s.fRamPreAlloc);
1933	Log(("pgmR3PhysRamPreAllocate: enter\n"));
1934
1935	/*
1936	* Walk the RAM ranges and allocate all RAM pages, halt at
1937	* the first allocation error.
1938	*/
1939	uint64_t cPages = 0;
1940	uint64_t NanoTS = RTTimeNanoTS();
1941	pgmLock(pVM);
1942	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1943	{
1944	PPGMPAGE pPage = &pRam->aPages[0];
1945	RTGCPHYS GCPhys = pRam->GCPhys;
1946	uint32_t cLeft = pRam->cb >> PAGE_SHIFT;
1947	while (cLeft-- > 0)
1948	{
1949	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1950	{
1951	switch (PGM_PAGE_GET_STATE(pPage))
1952	{
1953	case PGM_PAGE_STATE_ZERO:
1954	{
1955	int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
1956	if (RT_FAILURE(rc))
1957	{
1958	LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
1959	pgmUnlock(pVM);
1960	return rc;
1961	}
1962	cPages++;
1963	break;
1964	}
1965
1966	case PGM_PAGE_STATE_BALLOONED:
1967	case PGM_PAGE_STATE_ALLOCATED:
1968	case PGM_PAGE_STATE_WRITE_MONITORED:
1969	case PGM_PAGE_STATE_SHARED:
1970	/* nothing to do here. */
1971	break;
1972	}
1973	}
1974
1975	/* next */
1976	pPage++;
1977	GCPhys += PAGE_SIZE;
1978	}
1979	}
1980	pgmUnlock(pVM);
1981	NanoTS = RTTimeNanoTS() - NanoTS;
1982
1983	LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
1984	Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
1985	return VINF_SUCCESS;
1986	}
1987
1988
1989	/**
1990	* Checks shared page checksums.
1991	*
1992	* @param pVM The cross context VM structure.
1993	*/
1994	void pgmR3PhysAssertSharedPageChecksums(PVM pVM)
1995	{
1996	#ifdef VBOX_STRICT
1997	pgmLock(pVM);
1998
1999	if (pVM->pgm.s.cSharedPages > 0)
2000	{
2001	/*
2002	* Walk the ram ranges.
2003	*/
2004	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
2005	{
2006	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
2007	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
2008
2009	while (iPage-- > 0)
2010	{
2011	PPGMPAGE pPage = &pRam->aPages[iPage];
2012	if (PGM_PAGE_IS_SHARED(pPage))
2013	{
2014	uint32_t u32Checksum = pPage->s.u2Unused0/* \| ((uint32_t)pPage->s.u2Unused1 << 8)*/;
2015	if (!u32Checksum)
2016	{
2017	RTGCPHYS GCPhysPage = pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
2018	void const *pvPage;
2019	int rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhysPage, &pvPage);
2020	if (RT_SUCCESS(rc))
2021	{
2022	uint32_t u32Checksum2 = RTCrc32(pvPage, PAGE_SIZE);
2023	# if 0
2024	AssertMsg((u32Checksum2 & /UINT32_C(0x00000303)/ 0x3) == u32Checksum, ("GCPhysPage=%RGp\n", GCPhysPage));
2025	# else
2026	if ((u32Checksum2 & /UINT32_C(0x00000303)/ 0x3) == u32Checksum)
2027	LogFlow(("shpg %#x @ %RGp %#x [OK]\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
2028	else
2029	AssertMsgFailed(("shpg %#x @ %RGp %#x\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
2030	# endif
2031	}
2032	else
2033	AssertRC(rc);
2034	}
2035	}
2036
2037	} /* for each page */
2038
2039	} /* for each ram range */
2040	}
2041
2042	pgmUnlock(pVM);
2043	#endif /* VBOX_STRICT */
2044	NOREF(pVM);
2045	}
2046
2047
2048	/**
2049	* Resets the physical memory state.
2050	*
2051	* ASSUMES that the caller owns the PGM lock.
2052	*
2053	* @returns VBox status code.
2054	* @param pVM The cross context VM structure.
2055	*/
2056	int pgmR3PhysRamReset(PVM pVM)
2057	{
2058	PGM_LOCK_ASSERT_OWNER(pVM);
2059
2060	/* Reset the memory balloon. */
2061	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
2062	AssertRC(rc);
2063
2064	#ifdef VBOX_WITH_PAGE_SHARING
2065	/* Clear all registered shared modules. */
2066	pgmR3PhysAssertSharedPageChecksums(pVM);
2067	rc = GMMR3ResetSharedModules(pVM);
2068	AssertRC(rc);
2069	#endif
2070	/* Reset counters. */
2071	pVM->pgm.s.cReusedSharedPages = 0;
2072	pVM->pgm.s.cBalloonedPages = 0;
2073
2074	return VINF_SUCCESS;
2075	}
2076
2077
2078	/**
2079	* Resets (zeros) the RAM after all devices and components have been reset.
2080	*
2081	* ASSUMES that the caller owns the PGM lock.
2082	*
2083	* @returns VBox status code.
2084	* @param pVM The cross context VM structure.
2085	*/
2086	int pgmR3PhysRamZeroAll(PVM pVM)
2087	{
2088	PGM_LOCK_ASSERT_OWNER(pVM);
2089
2090	/*
2091	* We batch up pages that should be freed instead of calling GMM for
2092	* each and every one of them.
2093	*/
2094	uint32_t cPendingPages = 0;
2095	PGMMFREEPAGESREQ pReq;
2096	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2097	AssertLogRelRCReturn(rc, rc);
2098
2099	/*
2100	* Walk the ram ranges.
2101	*/
2102	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
2103	{
2104	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
2105	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
2106
2107	if ( !pVM->pgm.s.fRamPreAlloc
2108	&& pVM->pgm.s.fZeroRamPagesOnReset)
2109	{
2110	/* Replace all RAM pages by ZERO pages. */
2111	while (iPage-- > 0)
2112	{
2113	PPGMPAGE pPage = &pRam->aPages[iPage];
2114	switch (PGM_PAGE_GET_TYPE(pPage))
2115	{
2116	case PGMPAGETYPE_RAM:
2117	/* Do not replace pages part of a 2 MB continuous range
2118	with zero pages, but zero them instead. */
2119	if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE
2120	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
2121	{
2122	void *pvPage;
2123	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
2124	AssertLogRelRCReturn(rc, rc);
2125	ASMMemZeroPage(pvPage);
2126	}
2127	else if (PGM_PAGE_IS_BALLOONED(pPage))
2128	{
2129	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
2130	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
2131	}
2132	else if (!PGM_PAGE_IS_ZERO(pPage))
2133	{
2134	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
2135	PGMPAGETYPE_RAM);
2136	AssertLogRelRCReturn(rc, rc);
2137	}
2138	break;
2139
2140	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2141	case PGMPAGETYPE_SPECIAL_ALIAS_MMIO: /** @todo perhaps leave the special page alone? I don't think VT-x copes with this code. */
2142	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
2143	true /fDoAccounting/);
2144	break;
2145
2146	case PGMPAGETYPE_MMIO2:
2147	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
2148	case PGMPAGETYPE_ROM:
2149	case PGMPAGETYPE_MMIO:
2150	break;
2151	default:
2152	AssertFailed();
2153	}
2154	} /* for each page */
2155	}
2156	else
2157	{
2158	/* Zero the memory. */
2159	while (iPage-- > 0)
2160	{
2161	PPGMPAGE pPage = &pRam->aPages[iPage];
2162	switch (PGM_PAGE_GET_TYPE(pPage))
2163	{
2164	case PGMPAGETYPE_RAM:
2165	switch (PGM_PAGE_GET_STATE(pPage))
2166	{
2167	case PGM_PAGE_STATE_ZERO:
2168	break;
2169
2170	case PGM_PAGE_STATE_BALLOONED:
2171	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
2172	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
2173	break;
2174
2175	case PGM_PAGE_STATE_SHARED:
2176	case PGM_PAGE_STATE_WRITE_MONITORED:
2177	rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
2178	AssertLogRelRCReturn(rc, rc);
2179	RT_FALL_THRU();
2180
2181	case PGM_PAGE_STATE_ALLOCATED:
2182	if (pVM->pgm.s.fZeroRamPagesOnReset)
2183	{
2184	void *pvPage;
2185	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
2186	AssertLogRelRCReturn(rc, rc);
2187	ASMMemZeroPage(pvPage);
2188	}
2189	break;
2190	}
2191	break;
2192
2193	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2194	case PGMPAGETYPE_SPECIAL_ALIAS_MMIO: /** @todo perhaps leave the special page alone? I don't think VT-x copes with this code. */
2195	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
2196	true /fDoAccounting/);
2197	break;
2198
2199	case PGMPAGETYPE_MMIO2:
2200	case PGMPAGETYPE_ROM_SHADOW:
2201	case PGMPAGETYPE_ROM:
2202	case PGMPAGETYPE_MMIO:
2203	break;
2204	default:
2205	AssertFailed();
2206
2207	}
2208	} /* for each page */
2209	}
2210
2211	}
2212
2213	/*
2214	* Finish off any pages pending freeing.
2215	*/
2216	if (cPendingPages)
2217	{
2218	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2219	AssertLogRelRCReturn(rc, rc);
2220	}
2221	GMMR3FreePagesCleanup(pReq);
2222	return VINF_SUCCESS;
2223	}
2224
2225
2226	/**
2227	* Frees all RAM during VM termination
2228	*
2229	* ASSUMES that the caller owns the PGM lock.
2230	*
2231	* @returns VBox status code.
2232	* @param pVM The cross context VM structure.
2233	*/
2234	int pgmR3PhysRamTerm(PVM pVM)
2235	{
2236	PGM_LOCK_ASSERT_OWNER(pVM);
2237
2238	/* Reset the memory balloon. */
2239	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
2240	AssertRC(rc);
2241
2242	#ifdef VBOX_WITH_PAGE_SHARING
2243	/*
2244	* Clear all registered shared modules.
2245	*/
2246	pgmR3PhysAssertSharedPageChecksums(pVM);
2247	rc = GMMR3ResetSharedModules(pVM);
2248	AssertRC(rc);
2249
2250	/*
2251	* Flush the handy pages updates to make sure no shared pages are hiding
2252	* in there. (No unlikely if the VM shuts down, apparently.)
2253	*/
2254	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_FLUSH_HANDY_PAGES, 0, NULL);
2255	#endif
2256
2257	/*
2258	* We batch up pages that should be freed instead of calling GMM for
2259	* each and every one of them.
2260	*/
2261	uint32_t cPendingPages = 0;
2262	PGMMFREEPAGESREQ pReq;
2263	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2264	AssertLogRelRCReturn(rc, rc);
2265
2266	/*
2267	* Walk the ram ranges.
2268	*/
2269	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
2270	{
2271	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
2272	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
2273
2274	while (iPage-- > 0)
2275	{
2276	PPGMPAGE pPage = &pRam->aPages[iPage];
2277	switch (PGM_PAGE_GET_TYPE(pPage))
2278	{
2279	case PGMPAGETYPE_RAM:
2280	/* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
2281	/** @todo change this to explicitly free private pages here. */
2282	if (PGM_PAGE_IS_SHARED(pPage))
2283	{
2284	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
2285	PGMPAGETYPE_RAM);
2286	AssertLogRelRCReturn(rc, rc);
2287	}
2288	break;
2289
2290	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2291	case PGMPAGETYPE_SPECIAL_ALIAS_MMIO:
2292	case PGMPAGETYPE_MMIO2:
2293	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
2294	case PGMPAGETYPE_ROM:
2295	case PGMPAGETYPE_MMIO:
2296	break;
2297	default:
2298	AssertFailed();
2299	}
2300	} /* for each page */
2301	}
2302
2303	/*
2304	* Finish off any pages pending freeing.
2305	*/
2306	if (cPendingPages)
2307	{
2308	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2309	AssertLogRelRCReturn(rc, rc);
2310	}
2311	GMMR3FreePagesCleanup(pReq);
2312	return VINF_SUCCESS;
2313	}
2314
2315
2316	/**
2317	* This is the interface IOM is using to register an MMIO region.
2318	*
2319	* It will check for conflicts and ensure that a RAM range structure
2320	* is present before calling the PGMR3HandlerPhysicalRegister API to
2321	* register the callbacks.
2322	*
2323	* @returns VBox status code.
2324	*
2325	* @param pVM The cross context VM structure.
2326	* @param GCPhys The start of the MMIO region.
2327	* @param cb The size of the MMIO region.
2328	* @param hType The physical access handler type registration.
2329	* @param pvUserR3 The user argument for R3.
2330	* @param pvUserR0 The user argument for R0.
2331	* @param pvUserRC The user argument for RC.
2332	* @param pszDesc The description of the MMIO region.
2333	*/
2334	VMMR3DECL(int) PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMPHYSHANDLERTYPE hType,
2335	RTR3PTR pvUserR3, RTR0PTR pvUserR0, RTRCPTR pvUserRC, const char *pszDesc)
2336	{
2337	/*
2338	* Assert on some assumption.
2339	*/
2340	VM_ASSERT_EMT(pVM);
2341	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2342	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2343	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2344	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2345	Assert(((PPGMPHYSHANDLERTYPEINT)MMHyperHeapOffsetToPtr(pVM, hType))->enmKind == PGMPHYSHANDLERKIND_MMIO);
2346
2347	int rc = pgmLock(pVM);
2348	AssertRCReturn(rc, rc);
2349
2350	/*
2351	* Make sure there's a RAM range structure for the region.
2352	*/
2353	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2354	bool fRamExists = false;
2355	PPGMRAMRANGE pRamPrev = NULL;
2356	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2357	while (pRam && GCPhysLast >= pRam->GCPhys)
2358	{
2359	if ( GCPhysLast >= pRam->GCPhys
2360	&& GCPhys <= pRam->GCPhysLast)
2361	{
2362	/* Simplification: all within the same range. */
2363	AssertLogRelMsgReturnStmt( GCPhys >= pRam->GCPhys
2364	&& GCPhysLast <= pRam->GCPhysLast,
2365	("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
2366	GCPhys, GCPhysLast, pszDesc,
2367	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2368	pgmUnlock(pVM),
2369	VERR_PGM_RAM_CONFLICT);
2370
2371	/* Check that it's all RAM or MMIO pages. */
2372	PCPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2373	uint32_t cLeft = cb >> PAGE_SHIFT;
2374	while (cLeft-- > 0)
2375	{
2376	AssertLogRelMsgReturnStmt( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2377	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO,
2378	("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
2379	GCPhys, GCPhysLast, pszDesc, pRam->GCPhys, PGM_PAGE_GET_TYPE(pPage), pRam->pszDesc),
2380	pgmUnlock(pVM),
2381	VERR_PGM_RAM_CONFLICT);
2382	pPage++;
2383	}
2384
2385	/* Looks good. */
2386	fRamExists = true;
2387	break;
2388	}
2389
2390	/* next */
2391	pRamPrev = pRam;
2392	pRam = pRam->pNextR3;
2393	}
2394	PPGMRAMRANGE pNew;
2395	if (fRamExists)
2396	{
2397	pNew = NULL;
2398
2399	/*
2400	* Make all the pages in the range MMIO/ZERO pages, freeing any
2401	* RAM pages currently mapped here. This might not be 100% correct
2402	* for PCI memory, but we're doing the same thing for MMIO2 pages.
2403	*/
2404	rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
2405	AssertRCReturnStmt(rc, pgmUnlock(pVM), rc);
2406
2407	/* Force a PGM pool flush as guest ram references have been changed. */
2408	/** @todo not entirely SMP safe; assuming for now the guest takes
2409	* care of this internally (not touch mapped mmio while changing the
2410	* mapping). */
2411	PVMCPU pVCpu = VMMGetCpu(pVM);
2412	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2413	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2414	}
2415	else
2416	{
2417
2418	/*
2419	* No RAM range, insert an ad hoc one.
2420	*
2421	* Note that we don't have to tell REM about this range because
2422	* PGMHandlerPhysicalRegisterEx will do that for us.
2423	*/
2424	Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
2425
2426	const uint32_t cPages = cb >> PAGE_SHIFT;
2427	const size_t cbRamRange = RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]);
2428	rc = MMHyperAlloc(pVM, RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]), 16, MM_TAG_PGM_PHYS, (void **)&pNew);
2429	AssertLogRelMsgRCReturnStmt(rc, ("cbRamRange=%zu\n", cbRamRange), pgmUnlock(pVM), rc);
2430
2431	/* Initialize the range. */
2432	pNew->pSelfR0 = MMHyperCCToR0(pVM, pNew);
2433	pNew->GCPhys = GCPhys;
2434	pNew->GCPhysLast = GCPhysLast;
2435	pNew->cb = cb;
2436	pNew->pszDesc = pszDesc;
2437	pNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO;
2438	pNew->pvR3 = NULL;
2439	pNew->paLSPages = NULL;
2440
2441	uint32_t iPage = cPages;
2442	while (iPage-- > 0)
2443	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
2444	Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
2445
2446	/* update the page count stats. */
2447	pVM->pgm.s.cPureMmioPages += cPages;
2448	pVM->pgm.s.cAllPages += cPages;
2449
2450	/* link it */
2451	pgmR3PhysLinkRamRange(pVM, pNew, pRamPrev);
2452	}
2453
2454	/*
2455	* Register the access handler.
2456	*/
2457	rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, hType, pvUserR3, pvUserR0, pvUserRC, pszDesc);
2458	if ( RT_FAILURE(rc)
2459	&& !fRamExists)
2460	{
2461	pVM->pgm.s.cPureMmioPages -= cb >> PAGE_SHIFT;
2462	pVM->pgm.s.cAllPages -= cb >> PAGE_SHIFT;
2463
2464	/* remove the ad hoc range. */
2465	pgmR3PhysUnlinkRamRange2(pVM, pNew, pRamPrev);
2466	pNew->cb = pNew->GCPhys = pNew->GCPhysLast = NIL_RTGCPHYS;
2467	MMHyperFree(pVM, pRam);
2468	}
2469	pgmPhysInvalidatePageMapTLB(pVM);
2470
2471	pgmUnlock(pVM);
2472	return rc;
2473	}
2474
2475
2476	/**
2477	* This is the interface IOM is using to register an MMIO region.
2478	*
2479	* It will take care of calling PGMHandlerPhysicalDeregister and clean up
2480	* any ad hoc PGMRAMRANGE left behind.
2481	*
2482	* @returns VBox status code.
2483	* @param pVM The cross context VM structure.
2484	* @param GCPhys The start of the MMIO region.
2485	* @param cb The size of the MMIO region.
2486	*/
2487	VMMR3DECL(int) PGMR3PhysMMIODeregister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
2488	{
2489	VM_ASSERT_EMT(pVM);
2490
2491	int rc = pgmLock(pVM);
2492	AssertRCReturn(rc, rc);
2493
2494	/*
2495	* First deregister the handler, then check if we should remove the ram range.
2496	*/
2497	rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
2498	if (RT_SUCCESS(rc))
2499	{
2500	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2501	PPGMRAMRANGE pRamPrev = NULL;
2502	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2503	while (pRam && GCPhysLast >= pRam->GCPhys)
2504	{
2505	/** @todo We're being a bit too careful here. rewrite. */
2506	if ( GCPhysLast == pRam->GCPhysLast
2507	&& GCPhys == pRam->GCPhys)
2508	{
2509	Assert(pRam->cb == cb);
2510
2511	/*
2512	* See if all the pages are dead MMIO pages.
2513	*/
2514	uint32_t const cPages = cb >> PAGE_SHIFT;
2515	bool fAllMMIO = true;
2516	uint32_t iPage = 0;
2517	uint32_t cLeft = cPages;
2518	while (cLeft-- > 0)
2519	{
2520	PPGMPAGE pPage = &pRam->aPages[iPage];
2521	if ( !PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)
2522	/\|\| not-out-of-action later /)
2523	{
2524	fAllMMIO = false;
2525	AssertMsgFailed(("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2526	break;
2527	}
2528	Assert( PGM_PAGE_IS_ZERO(pPage)
2529	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
2530	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO);
2531	pPage++;
2532	}
2533	if (fAllMMIO)
2534	{
2535	/*
2536	* Ad-hoc range, unlink and free it.
2537	*/
2538	Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
2539	GCPhys, GCPhysLast, pRam->pszDesc));
2540
2541	pVM->pgm.s.cAllPages -= cPages;
2542	pVM->pgm.s.cPureMmioPages -= cPages;
2543
2544	pgmR3PhysUnlinkRamRange2(pVM, pRam, pRamPrev);
2545	pRam->cb = pRam->GCPhys = pRam->GCPhysLast = NIL_RTGCPHYS;
2546	MMHyperFree(pVM, pRam);
2547	break;
2548	}
2549	}
2550
2551	/*
2552	* Range match? It will all be within one range (see PGMAllHandler.cpp).
2553	*/
2554	if ( GCPhysLast >= pRam->GCPhys
2555	&& GCPhys <= pRam->GCPhysLast)
2556	{
2557	Assert(GCPhys >= pRam->GCPhys);
2558	Assert(GCPhysLast <= pRam->GCPhysLast);
2559
2560	/*
2561	* Turn the pages back into RAM pages.
2562	*/
2563	uint32_t iPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2564	uint32_t cLeft = cb >> PAGE_SHIFT;
2565	while (cLeft--)
2566	{
2567	PPGMPAGE pPage = &pRam->aPages[iPage];
2568	AssertMsg( (PGM_PAGE_IS_MMIO(pPage) && PGM_PAGE_IS_ZERO(pPage))
2569	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
2570	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO,
2571	("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2572	if (PGM_PAGE_IS_MMIO_OR_ALIAS(pPage))
2573	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_RAM);
2574	}
2575	break;
2576	}
2577
2578	/* next */
2579	pRamPrev = pRam;
2580	pRam = pRam->pNextR3;
2581	}
2582	}
2583
2584	/* Force a PGM pool flush as guest ram references have been changed. */
2585	/** @todo Not entirely SMP safe; assuming for now the guest takes care of
2586	* this internally (not touch mapped mmio while changing the mapping). */
2587	PVMCPU pVCpu = VMMGetCpu(pVM);
2588	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2589	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2590
2591	pgmPhysInvalidatePageMapTLB(pVM);
2592	pgmPhysInvalidRamRangeTlbs(pVM);
2593	pgmUnlock(pVM);
2594	return rc;
2595	}
2596
2597
2598	/**
2599	* Locate a MMIO2 range.
2600	*
2601	* @returns Pointer to the MMIO2 range.
2602	* @param pVM The cross context VM structure.
2603	* @param pDevIns The device instance owning the region.
2604	* @param iSubDev The sub-device number.
2605	* @param iRegion The region.
2606	*/
2607	DECLINLINE(PPGMREGMMIORANGE) pgmR3PhysMMIOExFind(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion)
2608	{
2609	/*
2610	* Search the list. There shouldn't be many entries.
2611	*/
2612	/** @todo Optimize this lookup! There may now be many entries and it'll
2613	* become really slow when doing MMR3HyperMapMMIO2 and similar. */
2614	for (PPGMREGMMIORANGE pCur = pVM->pgm.s.pRegMmioRangesR3; pCur; pCur = pCur->pNextR3)
2615	if ( pCur->pDevInsR3 == pDevIns
2616	&& pCur->iRegion == iRegion
2617	&& pCur->iSubDev == iSubDev)
2618	return pCur;
2619	return NULL;
2620	}
2621
2622
2623	#ifndef PGM_WITHOUT_MAPPINGS
2624	/**
2625	* @callback_method_impl{FNPGMRELOCATE, Relocate a floating MMIO/MMIO2 range.}
2626	* @sa pgmR3PhysRamRangeRelocate
2627	*/
2628	static DECLCALLBACK(bool) pgmR3PhysMMIOExRangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew,
2629	PGMRELOCATECALL enmMode, void *pvUser)
2630	{
2631	PPGMREGMMIORANGE pMmio = (PPGMREGMMIORANGE)pvUser;
2632	Assert(pMmio->RamRange.fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
2633	Assert(pMmio->RamRange.pSelfRC == GCPtrOld + PAGE_SIZE + RT_UOFFSETOF(PGMREGMMIORANGE, RamRange)); RT_NOREF_PV(GCPtrOld);
2634
2635	switch (enmMode)
2636	{
2637	case PGMRELOCATECALL_SUGGEST:
2638	return true;
2639
2640	case PGMRELOCATECALL_RELOCATE:
2641	{
2642	/*
2643	* Update myself, then relink all the ranges and flush the RC TLB.
2644	*/
2645	pgmLock(pVM);
2646
2647	pMmio->RamRange.pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE + RT_UOFFSETOF(PGMREGMMIORANGE, RamRange));
2648
2649	pgmR3PhysRelinkRamRanges(pVM);
2650	for (unsigned i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
2651	pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
2652
2653	pgmUnlock(pVM);
2654	return true;
2655	}
2656
2657	default:
2658	AssertFailedReturn(false);
2659	}
2660	}
2661	#endif /* !PGM_WITHOUT_MAPPINGS */
2662
2663
2664	/**
2665	* Calculates the number of chunks
2666	*
2667	* @returns Number of registration chunk needed.
2668	* @param pVM The cross context VM structure.
2669	* @param cb The size of the MMIO/MMIO2 range.
2670	* @param pcPagesPerChunk Where to return the number of pages tracked by each
2671	* chunk. Optional.
2672	* @param pcbChunk Where to return the guest mapping size for a chunk.
2673	*/
2674	static uint16_t pgmR3PhysMMIOExCalcChunkCount(PVM pVM, RTGCPHYS cb, uint32_t pcPagesPerChunk, uint32_t pcbChunk)
2675	{
2676	RT_NOREF_PV(pVM); /* without raw mode */
2677
2678	/*
2679	* This is the same calculation as PGMR3PhysRegisterRam does, except we'll be
2680	* needing a few bytes extra the PGMREGMMIORANGE structure.
2681	*
2682	* Note! In additions, we've got a 24 bit sub-page range for MMIO2 ranges, leaving
2683	* us with an absolute maximum of 16777215 pages per chunk (close to 64 GB).
2684	*/
2685	uint32_t cbChunk = 16U*_1M;
2686	uint32_t cPagesPerChunk = 1048048; /* max ~1048059 */
2687	AssertCompile(sizeof(PGMREGMMIORANGE) + sizeof(PGMPAGE) * 1048048 < 16U_1M - PAGE_SIZE 2);
2688	AssertRelease(cPagesPerChunk <= PGM_MMIO2_MAX_PAGE_COUNT); /* See above note. */
2689	AssertRelease(RT_UOFFSETOF_DYN(PGMREGMMIORANGE, RamRange.aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
2690	if (pcbChunk)
2691	*pcbChunk = cbChunk;
2692	if (pcPagesPerChunk)
2693	*pcPagesPerChunk = cPagesPerChunk;
2694
2695	/* Calc the number of chunks we need. */
2696	RTGCPHYS const cPages = cb >> X86_PAGE_SHIFT;
2697	uint16_t cChunks = (uint16_t)((cPages + cPagesPerChunk - 1) / cPagesPerChunk);
2698	AssertRelease((RTGCPHYS)cChunks * cPagesPerChunk >= cPages);
2699	return cChunks;
2700	}
2701
2702
2703	/**
2704	* Worker for PGMR3PhysMMIOExPreRegister & PGMR3PhysMMIO2Register that allocates
2705	* and the PGMREGMMIORANGE structures and does basic initialization.
2706	*
2707	* Caller must set type specfic members and initialize the PGMPAGE structures.
2708	*
2709	* @returns VBox status code.
2710	* @param pVM The cross context VM structure.
2711	* @param pDevIns The device instance owning the region.
2712	* @param iSubDev The sub-device number (internal PCI config number).
2713	* @param iRegion The region number. If the MMIO2 memory is a PCI
2714	* I/O region this number has to be the number of that
2715	* region. Otherwise it can be any number safe
2716	* UINT8_MAX.
2717	* @param cb The size of the region. Must be page aligned.
2718	* @param pszDesc The description.
2719	* @param ppHeadRet Where to return the pointer to the first
2720	* registration chunk.
2721	*
2722	* @thread EMT
2723	*/
2724	static int pgmR3PhysMMIOExCreate(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion, RTGCPHYS cb,
2725	const char pszDesc, PPGMREGMMIORANGE ppHeadRet)
2726	{
2727	/*
2728	* Figure out how many chunks we need and of which size.
2729	*/
2730	uint32_t cPagesPerChunk;
2731	uint16_t cChunks = pgmR3PhysMMIOExCalcChunkCount(pVM, cb, &cPagesPerChunk, NULL);
2732	AssertReturn(cChunks, VERR_PGM_PHYS_MMIO_EX_IPE);
2733
2734	/*
2735	* Allocate the chunks.
2736	*/
2737	PPGMREGMMIORANGE *ppNext = ppHeadRet;
2738	*ppNext = NULL;
2739
2740	int rc = VINF_SUCCESS;
2741	uint32_t cPagesLeft = cb >> X86_PAGE_SHIFT;
2742	for (uint16_t iChunk = 0; iChunk < cChunks && RT_SUCCESS(rc); iChunk++)
2743	{
2744	/*
2745	* We currently do a single RAM range for the whole thing. This will
2746	* probably have to change once someone needs really large MMIO regions,
2747	* as we will be running into SUPR3PageAllocEx limitations and such.
2748	*/
2749	const uint32_t cPagesTrackedByChunk = RT_MIN(cPagesLeft, cPagesPerChunk);
2750	const size_t cbRange = RT_UOFFSETOF_DYN(PGMREGMMIORANGE, RamRange.aPages[cPagesTrackedByChunk]);
2751	PPGMREGMMIORANGE pNew = NULL;
2752	if ( iChunk + 1 < cChunks
2753	\|\| cbRange >= _1M)
2754	{
2755	/*
2756	* Allocate memory for the registration structure.
2757	*/
2758	size_t const cChunkPages = RT_ALIGN_Z(cbRange, PAGE_SIZE) >> PAGE_SHIFT;
2759	size_t const cbChunk = (1 + cChunkPages + 1) << PAGE_SHIFT;
2760	AssertLogRelBreakStmt(cbChunk == (uint32_t)cbChunk, rc = VERR_OUT_OF_RANGE);
2761	PSUPPAGE paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
2762	AssertBreakStmt(paChunkPages, rc = VERR_NO_TMP_MEMORY);
2763	RTR0PTR R0PtrChunk = NIL_RTR0PTR;
2764	void *pvChunk = NULL;
2765	rc = SUPR3PageAllocEx(cChunkPages, 0 /fFlags/, &pvChunk, &R0PtrChunk, paChunkPages);
2766	AssertLogRelMsgRCBreakStmt(rc, ("rc=%Rrc, cChunkPages=%#zx\n", rc, cChunkPages), RTMemTmpFree(paChunkPages));
2767
2768	Assert(R0PtrChunk != NIL_RTR0PTR);
2769	memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
2770
2771	pNew = (PPGMREGMMIORANGE)pvChunk;
2772	pNew->RamRange.fFlags = PGM_RAM_RANGE_FLAGS_FLOATING;
2773	pNew->RamRange.pSelfR0 = R0PtrChunk + RT_UOFFSETOF(PGMREGMMIORANGE, RamRange);
2774
2775	RTMemTmpFree(paChunkPages);
2776	}
2777	/*
2778	* Not so big, do a one time hyper allocation.
2779	*/
2780	else
2781	{
2782	rc = MMR3HyperAllocOnceNoRel(pVM, cbRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
2783	AssertLogRelMsgRCBreak(rc, ("cbRange=%zu\n", cbRange));
2784
2785	/*
2786	* Initialize allocation specific items.
2787	*/
2788	//pNew->RamRange.fFlags = 0;
2789	pNew->RamRange.pSelfR0 = MMHyperCCToR0(pVM, &pNew->RamRange);
2790	}
2791
2792	/*
2793	* Initialize the registration structure (caller does specific bits).
2794	*/
2795	pNew->pDevInsR3 = pDevIns;
2796	//pNew->pvR3 = NULL;
2797	//pNew->pNext = NULL;
2798	//pNew->fFlags = 0;
2799	if (iChunk == 0)
2800	pNew->fFlags \|= PGMREGMMIORANGE_F_FIRST_CHUNK;
2801	if (iChunk + 1 == cChunks)
2802	pNew->fFlags \|= PGMREGMMIORANGE_F_LAST_CHUNK;
2803	pNew->iSubDev = iSubDev;
2804	pNew->iRegion = iRegion;
2805	pNew->idSavedState = UINT8_MAX;
2806	pNew->idMmio2 = UINT8_MAX;
2807	//pNew->pPhysHandlerR3 = NULL;
2808	//pNew->paLSPages = NULL;
2809	pNew->RamRange.GCPhys = NIL_RTGCPHYS;
2810	pNew->RamRange.GCPhysLast = NIL_RTGCPHYS;
2811	pNew->RamRange.pszDesc = pszDesc;
2812	pNew->RamRange.cb = pNew->cbReal = (RTGCPHYS)cPagesTrackedByChunk << X86_PAGE_SHIFT;
2813	pNew->RamRange.fFlags \|= PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX;
2814	//pNew->RamRange.pvR3 = NULL;
2815	//pNew->RamRange.paLSPages = NULL;
2816
2817	*ppNext = pNew;
2818	ASMCompilerBarrier();
2819	cPagesLeft -= cPagesTrackedByChunk;
2820	ppNext = &pNew->pNextR3;
2821	}
2822	Assert(cPagesLeft == 0);
2823
2824	if (RT_SUCCESS(rc))
2825	{
2826	Assert((*ppHeadRet)->fFlags & PGMREGMMIORANGE_F_FIRST_CHUNK);
2827	return VINF_SUCCESS;
2828	}
2829
2830	/*
2831	* Free floating ranges.
2832	*/
2833	while (*ppHeadRet)
2834	{
2835	PPGMREGMMIORANGE pFree = *ppHeadRet;
2836	*ppHeadRet = pFree->pNextR3;
2837
2838	if (pFree->RamRange.fFlags & PGM_RAM_RANGE_FLAGS_FLOATING)
2839	{
2840	const size_t cbRange = RT_UOFFSETOF_DYN(PGMREGMMIORANGE, RamRange.aPages[pFree->RamRange.cb >> X86_PAGE_SHIFT]);
2841	size_t const cChunkPages = RT_ALIGN_Z(cbRange, PAGE_SIZE) >> PAGE_SHIFT;
2842	SUPR3PageFreeEx(pFree, cChunkPages);
2843	}
2844	}
2845
2846	return rc;
2847	}
2848
2849
2850	/**
2851	* Common worker PGMR3PhysMMIOExPreRegister & PGMR3PhysMMIO2Register that links
2852	* a complete registration entry into the lists and lookup tables.
2853	*
2854	* @param pVM The cross context VM structure.
2855	* @param pNew The new MMIO / MMIO2 registration to link.
2856	*/
2857	static void pgmR3PhysMMIOExLink(PVM pVM, PPGMREGMMIORANGE pNew)
2858	{
2859	/*
2860	* Link it into the list (order doesn't matter, so insert it at the head).
2861	*
2862	* Note! The range we're link may consist of multiple chunks, so we have to
2863	* find the last one.
2864	*/
2865	PPGMREGMMIORANGE pLast = pNew;
2866	for (pLast = pNew; ; pLast = pLast->pNextR3)
2867	{
2868	if (pLast->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK)
2869	break;
2870	Assert(pLast->pNextR3);
2871	Assert(pLast->pNextR3->pDevInsR3 == pNew->pDevInsR3);
2872	Assert(pLast->pNextR3->iSubDev == pNew->iSubDev);
2873	Assert(pLast->pNextR3->iRegion == pNew->iRegion);
2874	Assert((pLast->pNextR3->fFlags & PGMREGMMIORANGE_F_MMIO2) == (pNew->fFlags & PGMREGMMIORANGE_F_MMIO2));
2875	Assert(pLast->pNextR3->idMmio2 == (pLast->fFlags & PGMREGMMIORANGE_F_MMIO2 ? pNew->idMmio2 + 1 : UINT8_MAX));
2876	}
2877
2878	pgmLock(pVM);
2879
2880	/* Link in the chain of ranges at the head of the list. */
2881	pLast->pNextR3 = pVM->pgm.s.pRegMmioRangesR3;
2882	pVM->pgm.s.pRegMmioRangesR3 = pNew;
2883
2884	/* If MMIO, insert the MMIO2 range/page IDs. */
2885	uint8_t idMmio2 = pNew->idMmio2;
2886	if (idMmio2 != UINT8_MAX)
2887	{
2888	for (;;)
2889	{
2890	Assert(pNew->fFlags & PGMREGMMIORANGE_F_MMIO2);
2891	Assert(pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] == NULL);
2892	Assert(pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] == NIL_RTR0PTR);
2893	pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] = pNew;
2894	pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] = pNew->RamRange.pSelfR0 - RT_UOFFSETOF(PGMREGMMIORANGE, RamRange);
2895	if (pNew->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK)
2896	break;
2897	pNew = pNew->pNextR3;
2898	}
2899	}
2900	else
2901	Assert(!(pNew->fFlags & PGMREGMMIORANGE_F_MMIO2));
2902
2903	pgmPhysInvalidatePageMapTLB(pVM);
2904	pgmUnlock(pVM);
2905	}
2906
2907
2908	/**
2909	* Allocate and pre-register an MMIO region.
2910	*
2911	* This is currently the way to deal with large MMIO regions. It may in the
2912	* future be extended to be the way we deal with all MMIO regions, but that
2913	* means we'll have to do something about the simple list based approach we take
2914	* to tracking the registrations.
2915	*
2916	* @returns VBox status code.
2917	* @retval VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
2918	* memory.
2919	* @retval VERR_ALREADY_EXISTS if the region already exists.
2920	*
2921	* @param pVM The cross context VM structure.
2922	* @param pDevIns The device instance owning the region.
2923	* @param iSubDev The sub-device number.
2924	* @param iRegion The region number. If the MMIO2 memory is a PCI
2925	* I/O region this number has to be the number of that
2926	* region. Otherwise it can be any number safe
2927	* UINT8_MAX.
2928	* @param cbRegion The size of the region. Must be page aligned.
2929	* @param hType The physical handler callback type.
2930	* @param pvUserR3 User parameter for ring-3 context callbacks.
2931	* @param pvUserR0 User parameter for ring-0 context callbacks.
2932	* @param pvUserRC User parameter for raw-mode context callbacks.
2933	* @param pszDesc The description.
2934	*
2935	* @thread EMT
2936	*
2937	* @sa PGMR3PhysMMIORegister, PGMR3PhysMMIO2Register,
2938	* PGMR3PhysMMIOExMap, PGMR3PhysMMIOExUnmap, PGMR3PhysMMIOExDeregister.
2939	*/
2940	VMMR3DECL(int) PGMR3PhysMMIOExPreRegister(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion, RTGCPHYS cbRegion,
2941	PGMPHYSHANDLERTYPE hType, RTR3PTR pvUserR3, RTR0PTR pvUserR0, RTRCPTR pvUserRC,
2942	const char *pszDesc)
2943	{
2944	/*
2945	* Validate input.
2946	*/
2947	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2948	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2949	AssertReturn(iSubDev <= UINT8_MAX, VERR_INVALID_PARAMETER);
2950	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2951	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2952	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2953	AssertReturn(pgmR3PhysMMIOExFind(pVM, pDevIns, iSubDev, iRegion) == NULL, VERR_ALREADY_EXISTS);
2954	AssertReturn(!(cbRegion & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2955	AssertReturn(cbRegion, VERR_INVALID_PARAMETER);
2956
2957	const uint32_t cPages = cbRegion >> PAGE_SHIFT;
2958	AssertLogRelReturn(((RTGCPHYS)cPages << PAGE_SHIFT) == cbRegion, VERR_INVALID_PARAMETER);
2959	AssertLogRelReturn(cPages <= (MM_MMIO_64_MAX >> X86_PAGE_SHIFT), VERR_OUT_OF_RANGE);
2960
2961	/*
2962	* For the 2nd+ instance, mangle the description string so it's unique.
2963	*/
2964	if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
2965	{
2966	pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
2967	if (!pszDesc)
2968	return VERR_NO_MEMORY;
2969	}
2970
2971	/*
2972	* Register the MMIO callbacks.
2973	*/
2974	PPGMPHYSHANDLER pPhysHandler;
2975	int rc = pgmHandlerPhysicalExCreate(pVM, hType, pvUserR3, pvUserR0, pvUserRC, pszDesc, &pPhysHandler);
2976	if (RT_SUCCESS(rc))
2977	{
2978	/*
2979	* Create the registered MMIO range record for it.
2980	*/
2981	PPGMREGMMIORANGE pNew;
2982	rc = pgmR3PhysMMIOExCreate(pVM, pDevIns, iSubDev, iRegion, cbRegion, pszDesc, &pNew);
2983	if (RT_SUCCESS(rc))
2984	{
2985	Assert(!(pNew->fFlags & PGMREGMMIORANGE_F_MMIO2));
2986
2987	/*
2988	* Intialize the page structures and set up physical handlers (one for each chunk).
2989	*/
2990	for (PPGMREGMMIORANGE pCur = pNew; pCur != NULL && RT_SUCCESS(rc); pCur = pCur->pNextR3)
2991	{
2992	if (pCur == pNew)
2993	pCur->pPhysHandlerR3 = pPhysHandler;
2994	else
2995	rc = pgmHandlerPhysicalExDup(pVM, pPhysHandler, &pCur->pPhysHandlerR3);
2996
2997	uint32_t iPage = pCur->RamRange.cb >> X86_PAGE_SHIFT;
2998	while (iPage-- > 0)
2999	PGM_PAGE_INIT_ZERO(&pCur->RamRange.aPages[iPage], pVM, PGMPAGETYPE_MMIO);
3000	}
3001	if (RT_SUCCESS(rc))
3002	{
3003	/*
3004	* Update the page count stats, link the registration and we're done.
3005	*/
3006	pVM->pgm.s.cAllPages += cPages;
3007	pVM->pgm.s.cPureMmioPages += cPages;
3008
3009	pgmR3PhysMMIOExLink(pVM, pNew);
3010	return VINF_SUCCESS;
3011	}
3012
3013	/*
3014	* Clean up in case we're out of memory for extra access handlers.
3015	*/
3016	while (pNew != NULL)
3017	{
3018	PPGMREGMMIORANGE pFree = pNew;
3019	pNew = pFree->pNextR3;
3020
3021	if (pFree->pPhysHandlerR3)
3022	{
3023	pgmHandlerPhysicalExDestroy(pVM, pFree->pPhysHandlerR3);
3024	pFree->pPhysHandlerR3 = NULL;
3025	}
3026
3027	if (pFree->RamRange.fFlags & PGM_RAM_RANGE_FLAGS_FLOATING)
3028	{
3029	const size_t cbRange = RT_UOFFSETOF_DYN(PGMREGMMIORANGE, RamRange.aPages[pFree->RamRange.cb >> X86_PAGE_SHIFT]);
3030	size_t const cChunkPages = RT_ALIGN_Z(cbRange, PAGE_SIZE) >> PAGE_SHIFT;
3031	SUPR3PageFreeEx(pFree, cChunkPages);
3032	}
3033	}
3034	}
3035	else
3036	pgmHandlerPhysicalExDestroy(pVM, pPhysHandler);
3037	}
3038	return rc;
3039	}
3040
3041
3042	/**
3043	* Allocate and register an MMIO2 region.
3044	*
3045	* As mentioned elsewhere, MMIO2 is just RAM spelled differently. It's RAM
3046	* associated with a device. It is also non-shared memory with a permanent
3047	* ring-3 mapping and page backing (presently).
3048	*
3049	* A MMIO2 range may overlap with base memory if a lot of RAM is configured for
3050	* the VM, in which case we'll drop the base memory pages. Presently we will
3051	* make no attempt to preserve anything that happens to be present in the base
3052	* memory that is replaced, this is of course incorrect but it's too much
3053	* effort.
3054	*
3055	* @returns VBox status code.
3056	* @retval VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
3057	* memory.
3058	* @retval VERR_ALREADY_EXISTS if the region already exists.
3059	*
3060	* @param pVM The cross context VM structure.
3061	* @param pDevIns The device instance owning the region.
3062	* @param iSubDev The sub-device number.
3063	* @param iRegion The region number. If the MMIO2 memory is a PCI
3064	* I/O region this number has to be the number of that
3065	* region. Otherwise it can be any number safe
3066	* UINT8_MAX.
3067	* @param cb The size of the region. Must be page aligned.
3068	* @param fFlags Reserved for future use, must be zero.
3069	* @param ppv Where to store the pointer to the ring-3 mapping of
3070	* the memory.
3071	* @param pszDesc The description.
3072	* @thread EMT
3073	*/
3074	VMMR3DECL(int) PGMR3PhysMMIO2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion, RTGCPHYS cb,
3075	uint32_t fFlags, void *ppv, const char pszDesc)
3076	{
3077	/*
3078	* Validate input.
3079	*/
3080	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3081	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3082	AssertReturn(iSubDev <= UINT8_MAX, VERR_INVALID_PARAMETER);
3083	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
3084	AssertPtrReturn(ppv, VERR_INVALID_POINTER);
3085	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
3086	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
3087	AssertReturn(pgmR3PhysMMIOExFind(pVM, pDevIns, iSubDev, iRegion) == NULL, VERR_ALREADY_EXISTS);
3088	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3089	AssertReturn(cb, VERR_INVALID_PARAMETER);
3090	AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
3091
3092	const uint32_t cPages = cb >> PAGE_SHIFT;
3093	AssertLogRelReturn(((RTGCPHYS)cPages << PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
3094	AssertLogRelReturn(cPages <= (MM_MMIO_64_MAX >> X86_PAGE_SHIFT), VERR_OUT_OF_RANGE);
3095
3096	/*
3097	* For the 2nd+ instance, mangle the description string so it's unique.
3098	*/
3099	if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
3100	{
3101	pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
3102	if (!pszDesc)
3103	return VERR_NO_MEMORY;
3104	}
3105
3106	/*
3107	* Allocate an MMIO2 range ID (not freed on failure).
3108	*
3109	* The zero ID is not used as it could be confused with NIL_GMM_PAGEID, so
3110	* the IDs goes from 1 thru PGM_MMIO2_MAX_RANGES.
3111	*/
3112	unsigned cChunks = pgmR3PhysMMIOExCalcChunkCount(pVM, cb, NULL, NULL);
3113	pgmLock(pVM);
3114	uint8_t idMmio2 = pVM->pgm.s.cMmio2Regions + 1;
3115	unsigned cNewMmio2Regions = pVM->pgm.s.cMmio2Regions + cChunks;
3116	if (cNewMmio2Regions > PGM_MMIO2_MAX_RANGES)
3117	{
3118	pgmUnlock(pVM);
3119	AssertLogRelFailedReturn(VERR_PGM_TOO_MANY_MMIO2_RANGES);
3120	}
3121	pVM->pgm.s.cMmio2Regions = cNewMmio2Regions;
3122	pgmUnlock(pVM);
3123
3124	/*
3125	* Try reserve and allocate the backing memory first as this is what is
3126	* most likely to fail.
3127	*/
3128	int rc = MMR3AdjustFixedReservation(pVM, cPages, pszDesc);
3129	if (RT_SUCCESS(rc))
3130	{
3131	PSUPPAGE paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(SUPPAGE));
3132	if (RT_SUCCESS(rc))
3133	{
3134	void *pvPages;
3135	rc = SUPR3PageAllocEx(cPages, 0 /fFlags/, &pvPages, NULL /pR0Ptr/, paPages);
3136	if (RT_SUCCESS(rc))
3137	{
3138	memset(pvPages, 0, cPages * PAGE_SIZE);
3139
3140	/*
3141	* Create the registered MMIO range record for it.
3142	*/
3143	PPGMREGMMIORANGE pNew;
3144	rc = pgmR3PhysMMIOExCreate(pVM, pDevIns, iSubDev, iRegion, cb, pszDesc, &pNew);
3145	if (RT_SUCCESS(rc))
3146	{
3147	uint32_t iSrcPage = 0;
3148	uint8_t pbCurPages = (uint8_t )pvPages;
3149	for (PPGMREGMMIORANGE pCur = pNew; pCur; pCur = pCur->pNextR3)
3150	{
3151	pCur->pvR3 = pbCurPages;
3152	pCur->RamRange.pvR3 = pbCurPages;
3153	pCur->idMmio2 = idMmio2;
3154	pCur->fFlags \|= PGMREGMMIORANGE_F_MMIO2;
3155
3156	uint32_t iDstPage = pCur->RamRange.cb >> X86_PAGE_SHIFT;
3157	while (iDstPage-- > 0)
3158	{
3159	PGM_PAGE_INIT(&pNew->RamRange.aPages[iDstPage],
3160	paPages[iDstPage + iSrcPage].Phys,
3161	PGM_MMIO2_PAGEID_MAKE(idMmio2, iDstPage),
3162	PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
3163	}
3164
3165	/* advance. */
3166	iSrcPage += pCur->RamRange.cb >> X86_PAGE_SHIFT;
3167	pbCurPages += pCur->RamRange.cb;
3168	idMmio2++;
3169	}
3170
3171	RTMemTmpFree(paPages);
3172
3173	/*
3174	* Update the page count stats, link the registration and we're done.
3175	*/
3176	pVM->pgm.s.cAllPages += cPages;
3177	pVM->pgm.s.cPrivatePages += cPages;
3178
3179	pgmR3PhysMMIOExLink(pVM, pNew);
3180
3181	*ppv = pvPages;
3182	return VINF_SUCCESS;
3183	}
3184
3185	SUPR3PageFreeEx(pvPages, cPages);
3186	}
3187	}
3188	RTMemTmpFree(paPages);
3189	MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pszDesc);
3190	}
3191	if (pDevIns->iInstance > 0)
3192	MMR3HeapFree((void *)pszDesc);
3193	return rc;
3194	}
3195
3196
3197	/**
3198	* Deregisters and frees an MMIO2 region or a pre-registered MMIO region
3199	*
3200	* Any physical (and virtual) access handlers registered for the region must
3201	* be deregistered before calling this function.
3202	*
3203	* @returns VBox status code.
3204	* @param pVM The cross context VM structure.
3205	* @param pDevIns The device instance owning the region.
3206	* @param iSubDev The sub-device number. Pass UINT32_MAX for wildcard
3207	* matching.
3208	* @param iRegion The region. Pass UINT32_MAX for wildcard matching.
3209	*/
3210	VMMR3DECL(int) PGMR3PhysMMIOExDeregister(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion)
3211	{
3212	/*
3213	* Validate input.
3214	*/
3215	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3216	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3217	AssertReturn(iSubDev <= UINT8_MAX \|\| iSubDev == UINT32_MAX, VERR_INVALID_PARAMETER);
3218	AssertReturn(iRegion <= UINT8_MAX \|\| iRegion == UINT32_MAX, VERR_INVALID_PARAMETER);
3219
3220	/*
3221	* The loop here scanning all registrations will make sure that multi-chunk ranges
3222	* get properly deregistered, though it's original purpose was the wildcard iRegion.
3223	*/
3224	pgmLock(pVM);
3225	int rc = VINF_SUCCESS;
3226	unsigned cFound = 0;
3227	PPGMREGMMIORANGE pPrev = NULL;
3228	PPGMREGMMIORANGE pCur = pVM->pgm.s.pRegMmioRangesR3;
3229	while (pCur)
3230	{
3231	if ( pCur->pDevInsR3 == pDevIns
3232	&& ( iRegion == UINT32_MAX
3233	\|\| pCur->iRegion == iRegion)
3234	&& ( iSubDev == UINT32_MAX
3235	\|\| pCur->iSubDev == iSubDev) )
3236	{
3237	cFound++;
3238
3239	/*
3240	* Unmap it if it's mapped.
3241	*/
3242	if (pCur->fFlags & PGMREGMMIORANGE_F_MAPPED)
3243	{
3244	int rc2 = PGMR3PhysMMIOExUnmap(pVM, pCur->pDevInsR3, pCur->iSubDev, pCur->iRegion, pCur->RamRange.GCPhys);
3245	AssertRC(rc2);
3246	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
3247	rc = rc2;
3248	}
3249
3250	/*
3251	* Must tell IOM about MMIO (first one only).
3252	*/
3253	if ((pCur->fFlags & (PGMREGMMIORANGE_F_MMIO2 \| PGMREGMMIORANGE_F_FIRST_CHUNK)) == PGMREGMMIORANGE_F_MMIO2)
3254	IOMR3MmioExNotifyDeregistered(pVM, pCur->pPhysHandlerR3->pvUserR3);
3255
3256	/*
3257	* Unlink it
3258	*/
3259	PPGMREGMMIORANGE pNext = pCur->pNextR3;
3260	if (pPrev)
3261	pPrev->pNextR3 = pNext;
3262	else
3263	pVM->pgm.s.pRegMmioRangesR3 = pNext;
3264	pCur->pNextR3 = NULL;
3265
3266	uint8_t idMmio2 = pCur->idMmio2;
3267	if (idMmio2 != UINT8_MAX)
3268	{
3269	Assert(pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] == pCur);
3270	pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] = NULL;
3271	pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] = NIL_RTR0PTR;
3272	}
3273
3274	/*
3275	* Free the memory.
3276	*/
3277	uint32_t const cPages = pCur->cbReal >> PAGE_SHIFT;
3278	if (pCur->fFlags & PGMREGMMIORANGE_F_MMIO2)
3279	{
3280	int rc2 = SUPR3PageFreeEx(pCur->pvR3, cPages);
3281	AssertRC(rc2);
3282	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
3283	rc = rc2;
3284
3285	rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pCur->RamRange.pszDesc);
3286	AssertRC(rc2);
3287	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
3288	rc = rc2;
3289	}
3290
3291	/* we're leaking hyper memory here if done at runtime. */
3292	#ifdef VBOX_STRICT
3293	VMSTATE const enmState = VMR3GetState(pVM);
3294	AssertMsg( enmState == VMSTATE_POWERING_OFF
3295	\|\| enmState == VMSTATE_POWERING_OFF_LS
3296	\|\| enmState == VMSTATE_OFF
3297	\|\| enmState == VMSTATE_OFF_LS
3298	\|\| enmState == VMSTATE_DESTROYING
3299	\|\| enmState == VMSTATE_TERMINATED
3300	\|\| enmState == VMSTATE_CREATING
3301	, ("%s\n", VMR3GetStateName(enmState)));
3302	#endif
3303
3304	const bool fIsMmio2 = RT_BOOL(pCur->fFlags & PGMREGMMIORANGE_F_MMIO2);
3305	if (pCur->RamRange.fFlags & PGM_RAM_RANGE_FLAGS_FLOATING)
3306	{
3307	const size_t cbRange = RT_UOFFSETOF_DYN(PGMREGMMIORANGE, RamRange.aPages[cPages]);
3308	size_t const cChunkPages = RT_ALIGN_Z(cbRange, PAGE_SIZE) >> PAGE_SHIFT;
3309	SUPR3PageFreeEx(pCur, cChunkPages);
3310	}
3311	/*else
3312	{
3313	rc = MMHyperFree(pVM, pCur); - does not work, see the alloc call.
3314	AssertRCReturn(rc, rc);
3315	} */
3316
3317
3318	/* update page count stats */
3319	pVM->pgm.s.cAllPages -= cPages;
3320	if (fIsMmio2)
3321	pVM->pgm.s.cPrivatePages -= cPages;
3322	else
3323	pVM->pgm.s.cPureMmioPages -= cPages;
3324
3325	/* next */
3326	pCur = pNext;
3327	}
3328	else
3329	{
3330	pPrev = pCur;
3331	pCur = pCur->pNextR3;
3332	}
3333	}
3334	pgmPhysInvalidatePageMapTLB(pVM);
3335	pgmUnlock(pVM);
3336	return !cFound && iRegion != UINT32_MAX && iSubDev != UINT32_MAX ? VERR_NOT_FOUND : rc;
3337	}
3338
3339
3340	/**
3341	* Maps a MMIO2 region or a pre-registered MMIO region.
3342	*
3343	* This is done when a guest / the bios / state loading changes the
3344	* PCI config. The replacing of base memory has the same restrictions
3345	* as during registration, of course.
3346	*
3347	* @returns VBox status code.
3348	*
3349	* @param pVM The cross context VM structure.
3350	* @param pDevIns The device instance owning the region.
3351	* @param iSubDev The sub-device number of the registered region.
3352	* @param iRegion The index of the registered region.
3353	* @param GCPhys The guest-physical address to be remapped.
3354	*/
3355	VMMR3DECL(int) PGMR3PhysMMIOExMap(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion, RTGCPHYS GCPhys)
3356	{
3357	/*
3358	* Validate input.
3359	*
3360	* Note! It's safe to walk the MMIO/MMIO2 list since registrations only
3361	* happens during VM construction.
3362	*/
3363	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3364	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3365	AssertReturn(iSubDev <= UINT8_MAX, VERR_INVALID_PARAMETER);
3366	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
3367	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
3368	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
3369	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3370
3371	PPGMREGMMIORANGE pFirstMmio = pgmR3PhysMMIOExFind(pVM, pDevIns, iSubDev, iRegion);
3372	AssertReturn(pFirstMmio, VERR_NOT_FOUND);
3373	Assert(pFirstMmio->fFlags & PGMREGMMIORANGE_F_FIRST_CHUNK);
3374
3375	PPGMREGMMIORANGE pLastMmio = pFirstMmio;
3376	RTGCPHYS cbRange = 0;
3377	for (;;)
3378	{
3379	AssertReturn(!(pLastMmio->fFlags & PGMREGMMIORANGE_F_MAPPED), VERR_WRONG_ORDER);
3380	Assert(pLastMmio->RamRange.GCPhys == NIL_RTGCPHYS);
3381	Assert(pLastMmio->RamRange.GCPhysLast == NIL_RTGCPHYS);
3382	Assert(pLastMmio->pDevInsR3 == pFirstMmio->pDevInsR3);
3383	Assert(pLastMmio->iSubDev == pFirstMmio->iSubDev);
3384	Assert(pLastMmio->iRegion == pFirstMmio->iRegion);
3385	cbRange += pLastMmio->RamRange.cb;
3386	if (pLastMmio->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK)
3387	break;
3388	pLastMmio = pLastMmio->pNextR3;
3389	}
3390
3391	RTGCPHYS GCPhysLast = GCPhys + cbRange - 1;
3392	AssertLogRelReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3393
3394	/*
3395	* Find our location in the ram range list, checking for restriction
3396	* we don't bother implementing yet (partially overlapping, multiple
3397	* ram ranges).
3398	*/
3399	pgmLock(pVM);
3400
3401	AssertReturnStmt(!(pFirstMmio->fFlags & PGMREGMMIORANGE_F_MAPPED), pgmUnlock(pVM), VERR_WRONG_ORDER);
3402
3403	bool fRamExists = false;
3404	PPGMRAMRANGE pRamPrev = NULL;
3405	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
3406	while (pRam && GCPhysLast >= pRam->GCPhys)
3407	{
3408	if ( GCPhys <= pRam->GCPhysLast
3409	&& GCPhysLast >= pRam->GCPhys)
3410	{
3411	/* Completely within? */
3412	AssertLogRelMsgReturnStmt( GCPhys >= pRam->GCPhys
3413	&& GCPhysLast <= pRam->GCPhysLast,
3414	("%RGp-%RGp (MMIOEx/%s) falls partly outside %RGp-%RGp (%s)\n",
3415	GCPhys, GCPhysLast, pFirstMmio->RamRange.pszDesc,
3416	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
3417	pgmUnlock(pVM),
3418	VERR_PGM_RAM_CONFLICT);
3419
3420	/* Check that all the pages are RAM pages. */
3421	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3422	uint32_t cPagesLeft = cbRange >> PAGE_SHIFT;
3423	while (cPagesLeft-- > 0)
3424	{
3425	AssertLogRelMsgReturnStmt(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
3426	("%RGp isn't a RAM page (%d) - mapping %RGp-%RGp (MMIO2/%s).\n",
3427	GCPhys, PGM_PAGE_GET_TYPE(pPage), GCPhys, GCPhysLast, pFirstMmio->RamRange.pszDesc),
3428	pgmUnlock(pVM),
3429	VERR_PGM_RAM_CONFLICT);
3430	pPage++;
3431	}
3432
3433	/* There can only be one MMIO/MMIO2 chunk matching here! */
3434	AssertLogRelMsgReturnStmt(pFirstMmio->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK,
3435	("%RGp-%RGp (MMIOEx/%s, flags %#X) consists of multiple chunks whereas the RAM somehow doesn't!\n",
3436	GCPhys, GCPhysLast, pFirstMmio->RamRange.pszDesc, pFirstMmio->fFlags),
3437	pgmUnlock(pVM),
3438	VERR_PGM_PHYS_MMIO_EX_IPE);
3439
3440	fRamExists = true;
3441	break;
3442	}
3443
3444	/* next */
3445	pRamPrev = pRam;
3446	pRam = pRam->pNextR3;
3447	}
3448	Log(("PGMR3PhysMMIOExMap: %RGp-%RGp fRamExists=%RTbool %s\n", GCPhys, GCPhysLast, fRamExists, pFirstMmio->RamRange.pszDesc));
3449
3450
3451	/*
3452	* Make the changes.
3453	*/
3454	RTGCPHYS GCPhysCur = GCPhys;
3455	for (PPGMREGMMIORANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
3456	{
3457	pCurMmio->RamRange.GCPhys = GCPhysCur;
3458	pCurMmio->RamRange.GCPhysLast = GCPhysCur + pCurMmio->RamRange.cb - 1;
3459	if (pCurMmio->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK)
3460	{
3461	Assert(pCurMmio->RamRange.GCPhysLast == GCPhysLast);
3462	break;
3463	}
3464	GCPhysCur += pCurMmio->RamRange.cb;
3465	}
3466
3467	if (fRamExists)
3468	{
3469	/*
3470	* Make all the pages in the range MMIO/ZERO pages, freeing any
3471	* RAM pages currently mapped here. This might not be 100% correct
3472	* for PCI memory, but we're doing the same thing for MMIO2 pages.
3473	*
3474	* We replace this MMIO/ZERO pages with real pages in the MMIO2 case.
3475	*/
3476	Assert(pFirstMmio->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK); /* Only one chunk */
3477
3478	int rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
3479	AssertRCReturnStmt(rc, pgmUnlock(pVM), rc);
3480
3481	if (pFirstMmio->fFlags & PGMREGMMIORANGE_F_MMIO2)
3482	{
3483	/* replace the pages, freeing all present RAM pages. */
3484	PPGMPAGE pPageSrc = &pFirstMmio->RamRange.aPages[0];
3485	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3486	uint32_t cPagesLeft = pFirstMmio->RamRange.cb >> PAGE_SHIFT;
3487	while (cPagesLeft-- > 0)
3488	{
3489	Assert(PGM_PAGE_IS_MMIO(pPageDst));
3490
3491	RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
3492	uint32_t const idPage = PGM_PAGE_GET_PAGEID(pPageSrc);
3493	PGM_PAGE_SET_PAGEID(pVM, pPageDst, idPage);
3494	PGM_PAGE_SET_HCPHYS(pVM, pPageDst, HCPhys);
3495	PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO2);
3496	PGM_PAGE_SET_STATE(pVM, pPageDst, PGM_PAGE_STATE_ALLOCATED);
3497	PGM_PAGE_SET_PDE_TYPE(pVM, pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
3498	PGM_PAGE_SET_PTE_INDEX(pVM, pPageDst, 0);
3499	PGM_PAGE_SET_TRACKING(pVM, pPageDst, 0);
3500	/* (We tell NEM at the end of the function.) */
3501
3502	pVM->pgm.s.cZeroPages--;
3503	GCPhys += PAGE_SIZE;
3504	pPageSrc++;
3505	pPageDst++;
3506	}
3507	}
3508
3509	/* Flush physical page map TLB. */
3510	pgmPhysInvalidatePageMapTLB(pVM);
3511
3512	/* Force a PGM pool flush as guest ram references have been changed. */
3513	/** @todo not entirely SMP safe; assuming for now the guest takes care of
3514	* this internally (not touch mapped mmio while changing the mapping). */
3515	PVMCPU pVCpu = VMMGetCpu(pVM);
3516	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3517	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3518	}
3519	else
3520	{
3521	/*
3522	* No RAM range, insert the ones prepared during registration.
3523	*/
3524	for (PPGMREGMMIORANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
3525	{
3526	/* Clear the tracking data of pages we're going to reactivate. */
3527	PPGMPAGE pPageSrc = &pCurMmio->RamRange.aPages[0];
3528	uint32_t cPagesLeft = pCurMmio->RamRange.cb >> PAGE_SHIFT;
3529	while (cPagesLeft-- > 0)
3530	{
3531	PGM_PAGE_SET_TRACKING(pVM, pPageSrc, 0);
3532	PGM_PAGE_SET_PTE_INDEX(pVM, pPageSrc, 0);
3533	pPageSrc++;
3534	}
3535
3536	/* link in the ram range */
3537	pgmR3PhysLinkRamRange(pVM, &pCurMmio->RamRange, pRamPrev);
3538
3539	if (pCurMmio->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK)
3540	{
3541	Assert(pCurMmio->RamRange.GCPhysLast == GCPhysLast);
3542	break;
3543	}
3544	pRamPrev = &pCurMmio->RamRange;
3545	}
3546	}
3547
3548	/*
3549	* Register the access handler if plain MMIO.
3550	*
3551	* We must register access handlers for each range since the access handler
3552	* code refuses to deal with multiple ranges (and we can).
3553	*/
3554	if (!(pFirstMmio->fFlags & PGMREGMMIORANGE_F_MMIO2))
3555	{
3556	int rc = VINF_SUCCESS;
3557	for (PPGMREGMMIORANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
3558	{
3559	Assert(!(pCurMmio->fFlags & PGMREGMMIORANGE_F_MAPPED));
3560	rc = pgmHandlerPhysicalExRegister(pVM, pCurMmio->pPhysHandlerR3, pCurMmio->RamRange.GCPhys,
3561	pCurMmio->RamRange.GCPhysLast);
3562	if (RT_FAILURE(rc))
3563	break;
3564	pCurMmio->fFlags \|= PGMREGMMIORANGE_F_MAPPED; /* Use this to mark that the handler is registered. */
3565	if (pCurMmio->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK)
3566	{
3567	rc = IOMR3MmioExNotifyMapped(pVM, pFirstMmio->pPhysHandlerR3->pvUserR3, GCPhys);
3568	break;
3569	}
3570	}
3571	if (RT_FAILURE(rc))
3572	{
3573	/* Almost impossible, but try clean up properly and get out of here. */
3574	for (PPGMREGMMIORANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
3575	{
3576	if (pCurMmio->fFlags & PGMREGMMIORANGE_F_MAPPED)
3577	{
3578	pCurMmio->fFlags &= ~PGMREGMMIORANGE_F_MAPPED;
3579	pgmHandlerPhysicalExDeregister(pVM, pCurMmio->pPhysHandlerR3, fRamExists);
3580	}
3581
3582	if (!fRamExists)
3583	pgmR3PhysUnlinkRamRange(pVM, &pCurMmio->RamRange);
3584	else
3585	{
3586	Assert(pCurMmio->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK); /* Only one chunk */
3587
3588	uint32_t cPagesLeft = pCurMmio->RamRange.cb >> PAGE_SHIFT;
3589	PPGMPAGE pPageDst = &pRam->aPages[(pCurMmio->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3590	while (cPagesLeft-- > 0)
3591	{
3592	PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
3593	pPageDst++;
3594	}
3595	}
3596
3597	pCurMmio->RamRange.GCPhys = NIL_RTGCPHYS;
3598	pCurMmio->RamRange.GCPhysLast = NIL_RTGCPHYS;
3599	if (pCurMmio->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK)
3600	break;
3601	}
3602
3603	pgmUnlock(pVM);
3604	return rc;
3605	}
3606	}
3607
3608	/*
3609	* We're good, set the flags and invalid the mapping TLB.
3610	*/
3611	for (PPGMREGMMIORANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
3612	{
3613	pCurMmio->fFlags \|= PGMREGMMIORANGE_F_MAPPED;
3614	if (fRamExists)
3615	pCurMmio->fFlags \|= PGMREGMMIORANGE_F_OVERLAPPING;
3616	else
3617	pCurMmio->fFlags &= ~PGMREGMMIORANGE_F_OVERLAPPING;
3618	if (pCurMmio->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK)
3619	break;
3620	}
3621	pgmPhysInvalidatePageMapTLB(pVM);
3622
3623	/*
3624	* Notify NEM while holding the lock (experimental) and REM without (like always).
3625	*/
3626	uint32_t const fNemNotify = (pFirstMmio->fFlags & PGMREGMMIORANGE_F_MMIO2 ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2 : 0)
3627	\| (pFirstMmio->fFlags & PGMREGMMIORANGE_F_OVERLAPPING ? NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE : 0);
3628	int rc = NEMR3NotifyPhysMmioExMap(pVM, GCPhys, cbRange, fNemNotify, pFirstMmio->pvR3);
3629
3630	pgmUnlock(pVM);
3631
3632	#ifdef VBOX_WITH_REM
3633	if (!fRamExists && (pFirstMmio->fFlags & PGMREGMMIORANGE_F_MMIO2)) /** @todo this doesn't look right. */
3634	REMR3NotifyPhysRamRegister(pVM, GCPhys, cbRange, REM_NOTIFY_PHYS_RAM_FLAGS_MMIO2);
3635	#endif
3636	return rc;
3637	}
3638
3639
3640	/**
3641	* Unmaps a MMIO2 or a pre-registered MMIO region.
3642	*
3643	* This is done when a guest / the bios / state loading changes the
3644	* PCI config. The replacing of base memory has the same restrictions
3645	* as during registration, of course.
3646	*/
3647	VMMR3DECL(int) PGMR3PhysMMIOExUnmap(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion, RTGCPHYS GCPhys)
3648	{
3649	/*
3650	* Validate input
3651	*/
3652	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3653	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3654	AssertReturn(iSubDev <= UINT8_MAX, VERR_INVALID_PARAMETER);
3655	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
3656	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
3657	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
3658	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3659
3660	PPGMREGMMIORANGE pFirstMmio = pgmR3PhysMMIOExFind(pVM, pDevIns, iSubDev, iRegion);
3661	AssertReturn(pFirstMmio, VERR_NOT_FOUND);
3662	Assert(pFirstMmio->fFlags & PGMREGMMIORANGE_F_FIRST_CHUNK);
3663
3664	PPGMREGMMIORANGE pLastMmio = pFirstMmio;
3665	RTGCPHYS cbRange = 0;
3666	for (;;)
3667	{
3668	AssertReturn(pLastMmio->fFlags & PGMREGMMIORANGE_F_MAPPED, VERR_WRONG_ORDER);
3669	AssertReturn(pLastMmio->RamRange.GCPhys == GCPhys + cbRange, VERR_INVALID_PARAMETER);
3670	Assert(pLastMmio->pDevInsR3 == pFirstMmio->pDevInsR3);
3671	Assert(pLastMmio->iSubDev == pFirstMmio->iSubDev);
3672	Assert(pLastMmio->iRegion == pFirstMmio->iRegion);
3673	cbRange += pLastMmio->RamRange.cb;
3674	if (pLastMmio->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK)
3675	break;
3676	pLastMmio = pLastMmio->pNextR3;
3677	}
3678
3679	Log(("PGMR3PhysMMIOExUnmap: %RGp-%RGp %s\n",
3680	pFirstMmio->RamRange.GCPhys, pLastMmio->RamRange.GCPhysLast, pFirstMmio->RamRange.pszDesc));
3681
3682	int rc = pgmLock(pVM);
3683	AssertRCReturn(rc, rc);
3684	uint16_t const fOldFlags = pFirstMmio->fFlags;
3685	AssertReturnStmt(fOldFlags & PGMREGMMIORANGE_F_MAPPED, pgmUnlock(pVM), VERR_WRONG_ORDER);
3686
3687	/*
3688	* If plain MMIO, we must deregister the handlers first.
3689	*/
3690	if (!(fOldFlags & PGMREGMMIORANGE_F_MMIO2))
3691	{
3692	PPGMREGMMIORANGE pCurMmio = pFirstMmio;
3693	rc = pgmHandlerPhysicalExDeregister(pVM, pFirstMmio->pPhysHandlerR3, RT_BOOL(fOldFlags & PGMREGMMIORANGE_F_OVERLAPPING));
3694	AssertRCReturnStmt(rc, pgmUnlock(pVM), rc);
3695	while (!(pCurMmio->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK))
3696	{
3697	pCurMmio = pCurMmio->pNextR3;
3698	rc = pgmHandlerPhysicalExDeregister(pVM, pCurMmio->pPhysHandlerR3, RT_BOOL(fOldFlags & PGMREGMMIORANGE_F_OVERLAPPING));
3699	AssertRCReturnStmt(rc, pgmUnlock(pVM), VERR_PGM_PHYS_MMIO_EX_IPE);
3700	}
3701
3702	IOMR3MmioExNotifyUnmapped(pVM, pFirstMmio->pPhysHandlerR3->pvUserR3, GCPhys);
3703	}
3704
3705	/*
3706	* Unmap it.
3707	*/
3708	RTGCPHYS const GCPhysRangeNotify = pFirstMmio->RamRange.GCPhys;
3709	if (fOldFlags & PGMREGMMIORANGE_F_OVERLAPPING)
3710	{
3711	/*
3712	* We've replaced RAM, replace with zero pages.
3713	*
3714	* Note! This is where we might differ a little from a real system, because
3715	* it's likely to just show the RAM pages as they were before the
3716	* MMIO/MMIO2 region was mapped here.
3717	*/
3718	/* Only one chunk allowed when overlapping! */
3719	Assert(fOldFlags & PGMREGMMIORANGE_F_LAST_CHUNK);
3720
3721	/* Restore the RAM pages we've replaced. */
3722	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
3723	while (pRam->GCPhys > pFirstMmio->RamRange.GCPhysLast)
3724	pRam = pRam->pNextR3;
3725
3726	uint32_t cPagesLeft = pFirstMmio->RamRange.cb >> PAGE_SHIFT;
3727	if (fOldFlags & PGMREGMMIORANGE_F_MMIO2)
3728	pVM->pgm.s.cZeroPages += cPagesLeft;
3729
3730	PPGMPAGE pPageDst = &pRam->aPages[(pFirstMmio->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3731	while (cPagesLeft-- > 0)
3732	{
3733	PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
3734	pPageDst++;
3735	}
3736
3737	/* Flush physical page map TLB. */
3738	pgmPhysInvalidatePageMapTLB(pVM);
3739
3740	/* Update range state. */
3741	pFirstMmio->RamRange.GCPhys = NIL_RTGCPHYS;
3742	pFirstMmio->RamRange.GCPhysLast = NIL_RTGCPHYS;
3743	pFirstMmio->fFlags &= ~(PGMREGMMIORANGE_F_OVERLAPPING \| PGMREGMMIORANGE_F_MAPPED);
3744	}
3745	else
3746	{
3747	/*
3748	* Unlink the chunks related to the MMIO/MMIO2 region.
3749	*/
3750	for (PPGMREGMMIORANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
3751	{
3752	pgmR3PhysUnlinkRamRange(pVM, &pCurMmio->RamRange);
3753	pCurMmio->RamRange.GCPhys = NIL_RTGCPHYS;
3754	pCurMmio->RamRange.GCPhysLast = NIL_RTGCPHYS;
3755	pCurMmio->fFlags &= ~(PGMREGMMIORANGE_F_OVERLAPPING \| PGMREGMMIORANGE_F_MAPPED);
3756	if (pCurMmio->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK)
3757	break;
3758	}
3759	}
3760
3761	/* Force a PGM pool flush as guest ram references have been changed. */
3762	/** @todo not entirely SMP safe; assuming for now the guest takes care
3763	* of this internally (not touch mapped mmio while changing the
3764	* mapping). */
3765	PVMCPU pVCpu = VMMGetCpu(pVM);
3766	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3767	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3768
3769	pgmPhysInvalidatePageMapTLB(pVM);
3770	pgmPhysInvalidRamRangeTlbs(pVM);
3771
3772	/*
3773	* Notify NEM while holding the lock (experimental) and REM without (like always).
3774	*/
3775	uint32_t const fNemFlags = (fOldFlags & PGMREGMMIORANGE_F_MMIO2 ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2 : 0)
3776	\| (fOldFlags & PGMREGMMIORANGE_F_OVERLAPPING ? NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE : 0);
3777	rc = NEMR3NotifyPhysMmioExUnmap(pVM, GCPhysRangeNotify, cbRange, fNemFlags);
3778	pgmUnlock(pVM);
3779	#ifdef VBOX_WITH_REM
3780	if ((fOldFlags & (PGMREGMMIORANGE_F_OVERLAPPING \| PGMREGMMIORANGE_F_MMIO2)) == PGMREGMMIORANGE_F_MMIO2)
3781	REMR3NotifyPhysRamDeregister(pVM, GCPhysRangeNotify, cbRange);
3782	#endif
3783	return rc;
3784	}
3785
3786
3787	/**
3788	* Reduces the mapping size of a MMIO2 or pre-registered MMIO region.
3789	*
3790	* This is mainly for dealing with old saved states after changing the default
3791	* size of a mapping region. See PGMDevHlpMMIOExReduce and
3792	* PDMPCIDEV::pfnRegionLoadChangeHookR3.
3793	*
3794	* The region must not currently be mapped when making this call. The VM state
3795	* must be state restore or VM construction.
3796	*
3797	* @returns VBox status code.
3798	* @param pVM The cross context VM structure.
3799	* @param pDevIns The device instance owning the region.
3800	* @param iSubDev The sub-device number of the registered region.
3801	* @param iRegion The index of the registered region.
3802	* @param cbRegion The new mapping size.
3803	*/
3804	VMMR3_INT_DECL(int) PGMR3PhysMMIOExReduce(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion, RTGCPHYS cbRegion)
3805	{
3806	/*
3807	* Validate input
3808	*/
3809	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3810	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3811	AssertReturn(iSubDev <= UINT8_MAX, VERR_INVALID_PARAMETER);
3812	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
3813	AssertReturn(cbRegion >= X86_PAGE_SIZE, VERR_INVALID_PARAMETER);
3814	AssertReturn(!(cbRegion & X86_PAGE_OFFSET_MASK), VERR_UNSUPPORTED_ALIGNMENT);
3815	VMSTATE enmVmState = VMR3GetState(pVM);
3816	AssertLogRelMsgReturn( enmVmState == VMSTATE_CREATING
3817	\|\| enmVmState == VMSTATE_LOADING,
3818	("enmVmState=%d (%s)\n", enmVmState, VMR3GetStateName(enmVmState)),
3819	VERR_VM_INVALID_VM_STATE);
3820
3821	int rc = pgmLock(pVM);
3822	AssertRCReturn(rc, rc);
3823
3824	PPGMREGMMIORANGE pFirstMmio = pgmR3PhysMMIOExFind(pVM, pDevIns, iSubDev, iRegion);
3825	if (pFirstMmio)
3826	{
3827	Assert(pFirstMmio->fFlags & PGMREGMMIORANGE_F_FIRST_CHUNK);
3828	if (!(pFirstMmio->fFlags & PGMREGMMIORANGE_F_MAPPED))
3829	{
3830	/*
3831	* NOTE! Current implementation does not support multiple ranges.
3832	* Implement when there is a real world need and thus a testcase.
3833	*/
3834	AssertLogRelMsgStmt(pFirstMmio->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK,
3835	("%s: %#x\n", pFirstMmio->RamRange.pszDesc, pFirstMmio->fFlags),
3836	rc = VERR_NOT_SUPPORTED);
3837	if (RT_SUCCESS(rc))
3838	{
3839	/*
3840	* Make the change.
3841	*/
3842	Log(("PGMR3PhysMMIOExReduce: %s changes from %RGp bytes (%RGp) to %RGp bytes.\n",
3843	pFirstMmio->RamRange.pszDesc, pFirstMmio->RamRange.cb, pFirstMmio->cbReal, cbRegion));
3844
3845	AssertLogRelMsgStmt(cbRegion <= pFirstMmio->cbReal,
3846	("%s: cbRegion=%#RGp cbReal=%#RGp\n", pFirstMmio->RamRange.pszDesc, cbRegion, pFirstMmio->cbReal),
3847	rc = VERR_OUT_OF_RANGE);
3848	if (RT_SUCCESS(rc))
3849	{
3850	pFirstMmio->RamRange.cb = cbRegion;
3851	}
3852	}
3853	}
3854	else
3855	rc = VERR_WRONG_ORDER;
3856	}
3857	else
3858	rc = VERR_NOT_FOUND;
3859
3860	pgmUnlock(pVM);
3861	return rc;
3862	}
3863
3864
3865	/**
3866	* Checks if the given address is an MMIO2 or pre-registered MMIO base address
3867	* or not.
3868	*
3869	* @returns true/false accordingly.
3870	* @param pVM The cross context VM structure.
3871	* @param pDevIns The owner of the memory, optional.
3872	* @param GCPhys The address to check.
3873	*/
3874	VMMR3DECL(bool) PGMR3PhysMMIOExIsBase(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
3875	{
3876	/*
3877	* Validate input
3878	*/
3879	VM_ASSERT_EMT_RETURN(pVM, false);
3880	AssertPtrReturn(pDevIns, false);
3881	AssertReturn(GCPhys != NIL_RTGCPHYS, false);
3882	AssertReturn(GCPhys != 0, false);
3883	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), false);
3884
3885	/*
3886	* Search the list.
3887	*/
3888	pgmLock(pVM);
3889	for (PPGMREGMMIORANGE pCurMmio = pVM->pgm.s.pRegMmioRangesR3; pCurMmio; pCurMmio = pCurMmio->pNextR3)
3890	if (pCurMmio->RamRange.GCPhys == GCPhys)
3891	{
3892	Assert(pCurMmio->fFlags & PGMREGMMIORANGE_F_MAPPED);
3893	bool fRet = RT_BOOL(pCurMmio->fFlags & PGMREGMMIORANGE_F_FIRST_CHUNK);
3894	pgmUnlock(pVM);
3895	return fRet;
3896	}
3897	pgmUnlock(pVM);
3898	return false;
3899	}
3900
3901
3902	/**
3903	* Gets the HC physical address of a page in the MMIO2 region.
3904	*
3905	* This is API is intended for MMHyper and shouldn't be called
3906	* by anyone else...
3907	*
3908	* @returns VBox status code.
3909	* @param pVM The cross context VM structure.
3910	* @param pDevIns The owner of the memory, optional.
3911	* @param iSubDev Sub-device number.
3912	* @param iRegion The region.
3913	* @param off The page expressed an offset into the MMIO2 region.
3914	* @param pHCPhys Where to store the result.
3915	*/
3916	VMMR3_INT_DECL(int) PGMR3PhysMMIO2GetHCPhys(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion,
3917	RTGCPHYS off, PRTHCPHYS pHCPhys)
3918	{
3919	/*
3920	* Validate input
3921	*/
3922	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3923	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3924	AssertReturn(iSubDev <= UINT8_MAX, VERR_INVALID_PARAMETER);
3925	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
3926
3927	pgmLock(pVM);
3928	PPGMREGMMIORANGE pCurMmio = pgmR3PhysMMIOExFind(pVM, pDevIns, iSubDev, iRegion);
3929	AssertReturn(pCurMmio, VERR_NOT_FOUND);
3930	AssertReturn(pCurMmio->fFlags & (PGMREGMMIORANGE_F_MMIO2 \| PGMREGMMIORANGE_F_FIRST_CHUNK), VERR_WRONG_TYPE);
3931
3932	while ( off >= pCurMmio->RamRange.cb
3933	&& !(pCurMmio->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK))
3934	{
3935	off -= pCurMmio->RamRange.cb;
3936	pCurMmio = pCurMmio->pNextR3;
3937	}
3938	AssertReturn(off < pCurMmio->RamRange.cb, VERR_INVALID_PARAMETER);
3939
3940	PCPGMPAGE pPage = &pCurMmio->RamRange.aPages[off >> PAGE_SHIFT];
3941	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3942	pgmUnlock(pVM);
3943	return VINF_SUCCESS;
3944	}
3945
3946
3947	/**
3948	* Maps a portion of an MMIO2 region into kernel space (host).
3949	*
3950	* The kernel mapping will become invalid when the MMIO2 memory is deregistered
3951	* or the VM is terminated.
3952	*
3953	* @return VBox status code.
3954	*
3955	* @param pVM The cross context VM structure.
3956	* @param pDevIns The device owning the MMIO2 memory.
3957	* @param iSubDev The sub-device number.
3958	* @param iRegion The region.
3959	* @param off The offset into the region. Must be page aligned.
3960	* @param cb The number of bytes to map. Must be page aligned.
3961	* @param pszDesc Mapping description.
3962	* @param pR0Ptr Where to store the R0 address.
3963	*/
3964	VMMR3_INT_DECL(int) PGMR3PhysMMIO2MapKernel(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion,
3965	RTGCPHYS off, RTGCPHYS cb, const char *pszDesc, PRTR0PTR pR0Ptr)
3966	{
3967	/*
3968	* Validate input.
3969	*/
3970	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3971	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3972	AssertReturn(iSubDev <= UINT8_MAX, VERR_INVALID_PARAMETER);
3973	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
3974
3975	PPGMREGMMIORANGE pFirstRegMmio = pgmR3PhysMMIOExFind(pVM, pDevIns, iSubDev, iRegion);
3976	AssertReturn(pFirstRegMmio, VERR_NOT_FOUND);
3977	AssertReturn(pFirstRegMmio->fFlags & (PGMREGMMIORANGE_F_MMIO2 \| PGMREGMMIORANGE_F_FIRST_CHUNK), VERR_WRONG_TYPE);
3978	AssertReturn(off < pFirstRegMmio->RamRange.cb, VERR_INVALID_PARAMETER);
3979	AssertReturn(cb <= pFirstRegMmio->RamRange.cb, VERR_INVALID_PARAMETER);
3980	AssertReturn(off + cb <= pFirstRegMmio->RamRange.cb, VERR_INVALID_PARAMETER);
3981	NOREF(pszDesc);
3982
3983	/*
3984	* Pass the request on to the support library/driver.
3985	*/
3986	#if defined(RT_OS_WINDOWS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_OS2) /** @todo Fully implement RTR0MemObjMapKernelEx everywhere. */
3987	AssertLogRelReturn(off == 0, VERR_NOT_SUPPORTED);
3988	AssertLogRelReturn(pFirstRegMmio->fFlags & PGMREGMMIORANGE_F_LAST_CHUNK, VERR_NOT_SUPPORTED);
3989	int rc = SUPR3PageMapKernel(pFirstRegMmio->pvR3, 0 /off/, pFirstRegMmio->RamRange.cb, 0 /fFlags/, pR0Ptr);
3990	#else
3991	int rc = SUPR3PageMapKernel(pFirstRegMmio->pvR3, off, cb, 0 /fFlags/, pR0Ptr);
3992	#endif
3993
3994	return rc;
3995	}
3996
3997
3998	/**
3999	* Changes the region number of an MMIO2 or pre-registered MMIO region.
4000	*
4001	* This is only for dealing with save state issues, nothing else.
4002	*
4003	* @return VBox status code.
4004	*
4005	* @param pVM The cross context VM structure.
4006	* @param pDevIns The device owning the MMIO2 memory.
4007	* @param iSubDev The sub-device number.
4008	* @param iRegion The region.
4009	* @param iNewRegion The new region index.
4010	*
4011	* @sa @bugref{9359}
4012	*/
4013	VMMR3_INT_DECL(int) PGMR3PhysMMIOExChangeRegionNo(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion,
4014	uint32_t iNewRegion)
4015	{
4016	/*
4017	* Validate input.
4018	*/
4019	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
4020	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
4021	AssertReturn(iSubDev <= UINT8_MAX, VERR_INVALID_PARAMETER);
4022	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
4023	AssertReturn(iNewRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
4024
4025	AssertReturn(pVM->enmVMState == VMSTATE_LOADING, VERR_INVALID_STATE);
4026
4027	PPGMREGMMIORANGE pFirstRegMmio = pgmR3PhysMMIOExFind(pVM, pDevIns, iSubDev, iRegion);
4028	AssertReturn(pFirstRegMmio, VERR_NOT_FOUND);
4029	AssertReturn(pgmR3PhysMMIOExFind(pVM, pDevIns, iSubDev, iNewRegion) == NULL, VERR_RESOURCE_IN_USE);
4030
4031	/*
4032	* Make the change.
4033	*/
4034	pFirstRegMmio->iRegion = (uint8_t)iNewRegion;
4035
4036	return VINF_SUCCESS;
4037	}
4038
4039
4040	/**
4041	* Worker for PGMR3PhysRomRegister.
4042	*
4043	* This is here to simplify lock management, i.e. the caller does all the
4044	* locking and we can simply return without needing to remember to unlock
4045	* anything first.
4046	*
4047	* @returns VBox status code.
4048	* @param pVM The cross context VM structure.
4049	* @param pDevIns The device instance owning the ROM.
4050	* @param GCPhys First physical address in the range.
4051	* Must be page aligned!
4052	* @param cb The size of the range (in bytes).
4053	* Must be page aligned!
4054	* @param pvBinary Pointer to the binary data backing the ROM image.
4055	* @param cbBinary The size of the binary data pvBinary points to.
4056	* This must be less or equal to @a cb.
4057	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
4058	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
4059	* @param pszDesc Pointer to description string. This must not be freed.
4060	*/
4061	static int pgmR3PhysRomRegisterLocked(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
4062	const void pvBinary, uint32_t cbBinary, uint32_t fFlags, const char pszDesc)
4063	{
4064	/*
4065	* Validate input.
4066	*/
4067	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
4068	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
4069	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
4070	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
4071	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
4072	AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
4073	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
4074	AssertReturn(!(fFlags & ~(PGMPHYS_ROM_FLAGS_SHADOWED \| PGMPHYS_ROM_FLAGS_PERMANENT_BINARY)), VERR_INVALID_PARAMETER);
4075	VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
4076
4077	const uint32_t cPages = cb >> PAGE_SHIFT;
4078
4079	/*
4080	* Find the ROM location in the ROM list first.
4081	*/
4082	PPGMROMRANGE pRomPrev = NULL;
4083	PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3;
4084	while (pRom && GCPhysLast >= pRom->GCPhys)
4085	{
4086	if ( GCPhys <= pRom->GCPhysLast
4087	&& GCPhysLast >= pRom->GCPhys)
4088	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
4089	GCPhys, GCPhysLast, pszDesc,
4090	pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
4091	VERR_PGM_RAM_CONFLICT);
4092	/* next */
4093	pRomPrev = pRom;
4094	pRom = pRom->pNextR3;
4095	}
4096
4097	/*
4098	* Find the RAM location and check for conflicts.
4099	*
4100	* Conflict detection is a bit different than for RAM
4101	* registration since a ROM can be located within a RAM
4102	* range. So, what we have to check for is other memory
4103	* types (other than RAM that is) and that we don't span
4104	* more than one RAM range (layz).
4105	*/
4106	bool fRamExists = false;
4107	PPGMRAMRANGE pRamPrev = NULL;
4108	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
4109	while (pRam && GCPhysLast >= pRam->GCPhys)
4110	{
4111	if ( GCPhys <= pRam->GCPhysLast
4112	&& GCPhysLast >= pRam->GCPhys)
4113	{
4114	/* completely within? */
4115	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
4116	&& GCPhysLast <= pRam->GCPhysLast,
4117	("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
4118	GCPhys, GCPhysLast, pszDesc,
4119	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
4120	VERR_PGM_RAM_CONFLICT);
4121	fRamExists = true;
4122	break;
4123	}
4124
4125	/* next */
4126	pRamPrev = pRam;
4127	pRam = pRam->pNextR3;
4128	}
4129	if (fRamExists)
4130	{
4131	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
4132	uint32_t cPagesLeft = cPages;
4133	while (cPagesLeft-- > 0)
4134	{
4135	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
4136	("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
4137	pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << PAGE_SHIFT),
4138	pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
4139	Assert(PGM_PAGE_IS_ZERO(pPage));
4140	pPage++;
4141	}
4142	}
4143
4144	/*
4145	* Update the base memory reservation if necessary.
4146	*/
4147	uint32_t cExtraBaseCost = fRamExists ? 0 : cPages;
4148	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
4149	cExtraBaseCost += cPages;
4150	if (cExtraBaseCost)
4151	{
4152	int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
4153	if (RT_FAILURE(rc))
4154	return rc;
4155	}
4156
4157	/*
4158	* Allocate memory for the virgin copy of the RAM.
4159	*/
4160	PGMMALLOCATEPAGESREQ pReq;
4161	int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
4162	AssertRCReturn(rc, rc);
4163
4164	for (uint32_t iPage = 0; iPage < cPages; iPage++)
4165	{
4166	pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
4167	pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
4168	pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
4169	}
4170
4171	rc = GMMR3AllocatePagesPerform(pVM, pReq);
4172	if (RT_FAILURE(rc))
4173	{
4174	GMMR3AllocatePagesCleanup(pReq);
4175	return rc;
4176	}
4177
4178	/*
4179	* Allocate the new ROM range and RAM range (if necessary).
4180	*/
4181	PPGMROMRANGE pRomNew;
4182	rc = MMHyperAlloc(pVM, RT_UOFFSETOF_DYN(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
4183	if (RT_SUCCESS(rc))
4184	{
4185	PPGMRAMRANGE pRamNew = NULL;
4186	if (!fRamExists)
4187	rc = MMHyperAlloc(pVM, RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]), sizeof(PGMPAGE), MM_TAG_PGM_PHYS, (void **)&pRamNew);
4188	if (RT_SUCCESS(rc))
4189	{
4190	/*
4191	* Initialize and insert the RAM range (if required).
4192	*/
4193	PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
4194	if (!fRamExists)
4195	{
4196	pRamNew->pSelfR0 = MMHyperCCToR0(pVM, pRamNew);
4197	pRamNew->GCPhys = GCPhys;
4198	pRamNew->GCPhysLast = GCPhysLast;
4199	pRamNew->cb = cb;
4200	pRamNew->pszDesc = pszDesc;
4201	pRamNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_ROM;
4202	pRamNew->pvR3 = NULL;
4203	pRamNew->paLSPages = NULL;
4204
4205	PPGMPAGE pPage = &pRamNew->aPages[0];
4206	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
4207	{
4208	PGM_PAGE_INIT(pPage,
4209	pReq->aPages[iPage].HCPhysGCPhys,
4210	pReq->aPages[iPage].idPage,
4211	PGMPAGETYPE_ROM,
4212	PGM_PAGE_STATE_ALLOCATED);
4213
4214	pRomPage->Virgin = *pPage;
4215	}
4216
4217	pVM->pgm.s.cAllPages += cPages;
4218	pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
4219	}
4220	else
4221	{
4222	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
4223	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
4224	{
4225	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_ROM);
4226	PGM_PAGE_SET_HCPHYS(pVM, pPage, pReq->aPages[iPage].HCPhysGCPhys);
4227	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
4228	PGM_PAGE_SET_PAGEID(pVM, pPage, pReq->aPages[iPage].idPage);
4229	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
4230	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4231	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4232
4233	pRomPage->Virgin = *pPage;
4234	}
4235
4236	pRamNew = pRam;
4237
4238	pVM->pgm.s.cZeroPages -= cPages;
4239	}
4240	pVM->pgm.s.cPrivatePages += cPages;
4241
4242	/* Flush physical page map TLB. */
4243	pgmPhysInvalidatePageMapTLB(pVM);
4244
4245
4246	/* Notify NEM before we register handlers. */
4247	uint32_t const fNemNotify = (fRamExists ? NEM_NOTIFY_PHYS_ROM_F_REPLACE : 0)
4248	\| (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED ? NEM_NOTIFY_PHYS_ROM_F_SHADOW : 0);
4249	rc = NEMR3NotifyPhysRomRegisterEarly(pVM, GCPhys, cb, fNemNotify);
4250
4251	/*
4252	* !HACK ALERT! REM + (Shadowed) ROM ==> mess.
4253	*
4254	* If it's shadowed we'll register the handler after the ROM notification
4255	* so we get the access handler callbacks that we should. If it isn't
4256	* shadowed we'll do it the other way around to make REM use the built-in
4257	* ROM behavior and not the handler behavior (which is to route all access
4258	* to PGM atm).
4259	*/
4260	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
4261	{
4262	#ifdef VBOX_WITH_REM
4263	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, true /* fShadowed */);
4264	#endif
4265	if (RT_SUCCESS(rc))
4266	rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, pVM->pgm.s.hRomPhysHandlerType,
4267	pRomNew, MMHyperCCToR0(pVM, pRomNew), MMHyperCCToRC(pVM, pRomNew),
4268	pszDesc);
4269	}
4270	else
4271	{
4272	if (RT_SUCCESS(rc))
4273	rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, pVM->pgm.s.hRomPhysHandlerType,
4274	pRomNew, MMHyperCCToR0(pVM, pRomNew), MMHyperCCToRC(pVM, pRomNew),
4275	pszDesc);
4276	#ifdef VBOX_WITH_REM
4277	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, false /* fShadowed */);
4278	#endif
4279	}
4280	if (RT_SUCCESS(rc))
4281	{
4282	/*
4283	* Copy the image over to the virgin pages.
4284	* This must be done after linking in the RAM range.
4285	*/
4286	size_t cbBinaryLeft = cbBinary;
4287	PPGMPAGE pRamPage = &pRamNew->aPages[(GCPhys - pRamNew->GCPhys) >> PAGE_SHIFT];
4288	for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
4289	{
4290	void *pvDstPage;
4291	rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << PAGE_SHIFT), &pvDstPage);
4292	if (RT_FAILURE(rc))
4293	{
4294	VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
4295	break;
4296	}
4297	if (cbBinaryLeft >= PAGE_SIZE)
4298	{
4299	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), PAGE_SIZE);
4300	cbBinaryLeft -= PAGE_SIZE;
4301	}
4302	else
4303	{
4304	ASMMemZeroPage(pvDstPage); /* (shouldn't be necessary, but can't hurt either) */
4305	if (cbBinaryLeft > 0)
4306	{
4307	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), cbBinaryLeft);
4308	cbBinaryLeft = 0;
4309	}
4310	}
4311	}
4312	if (RT_SUCCESS(rc))
4313	{
4314	/*
4315	* Initialize the ROM range.
4316	* Note that the Virgin member of the pages has already been initialized above.
4317	*/
4318	pRomNew->GCPhys = GCPhys;
4319	pRomNew->GCPhysLast = GCPhysLast;
4320	pRomNew->cb = cb;
4321	pRomNew->fFlags = fFlags;
4322	pRomNew->idSavedState = UINT8_MAX;
4323	pRomNew->cbOriginal = cbBinary;
4324	pRomNew->pszDesc = pszDesc;
4325	pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
4326	? pvBinary : RTMemDup(pvBinary, cbBinary);
4327	if (pRomNew->pvOriginal)
4328	{
4329	for (unsigned iPage = 0; iPage < cPages; iPage++)
4330	{
4331	PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
4332	pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
4333	PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
4334	}
4335
4336	/* update the page count stats for the shadow pages. */
4337	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
4338	{
4339	pVM->pgm.s.cZeroPages += cPages;
4340	pVM->pgm.s.cAllPages += cPages;
4341	}
4342
4343	/*
4344	* Insert the ROM range, tell REM and return successfully.
4345	*/
4346	pRomNew->pNextR3 = pRom;
4347	pRomNew->pNextR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
4348
4349	if (pRomPrev)
4350	{
4351	pRomPrev->pNextR3 = pRomNew;
4352	pRomPrev->pNextR0 = MMHyperCCToR0(pVM, pRomNew);
4353	}
4354	else
4355	{
4356	pVM->pgm.s.pRomRangesR3 = pRomNew;
4357	pVM->pgm.s.pRomRangesR0 = MMHyperCCToR0(pVM, pRomNew);
4358	}
4359
4360	pgmPhysInvalidatePageMapTLB(pVM);
4361	GMMR3AllocatePagesCleanup(pReq);
4362
4363	/* Notify NEM again. */
4364	return NEMR3NotifyPhysRomRegisterLate(pVM, GCPhys, cb, fNemNotify);
4365	}
4366
4367	/* bail out */
4368	rc = VERR_NO_MEMORY;
4369	}
4370
4371	int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
4372	AssertRC(rc2);
4373	}
4374
4375	if (!fRamExists)
4376	{
4377	pgmR3PhysUnlinkRamRange2(pVM, pRamNew, pRamPrev);
4378	MMHyperFree(pVM, pRamNew);
4379	}
4380	}
4381	MMHyperFree(pVM, pRomNew);
4382	}
4383
4384	/** @todo Purge the mapping cache or something... */
4385	GMMR3FreeAllocatedPages(pVM, pReq);
4386	GMMR3AllocatePagesCleanup(pReq);
4387	return rc;
4388	}
4389
4390
4391	/**
4392	* Registers a ROM image.
4393	*
4394	* Shadowed ROM images requires double the amount of backing memory, so,
4395	* don't use that unless you have to. Shadowing of ROM images is process
4396	* where we can select where the reads go and where the writes go. On real
4397	* hardware the chipset provides means to configure this. We provide
4398	* PGMR3PhysProtectROM() for this purpose.
4399	*
4400	* A read-only copy of the ROM image will always be kept around while we
4401	* will allocate RAM pages for the changes on demand (unless all memory
4402	* is configured to be preallocated).
4403	*
4404	* @returns VBox status code.
4405	* @param pVM The cross context VM structure.
4406	* @param pDevIns The device instance owning the ROM.
4407	* @param GCPhys First physical address in the range.
4408	* Must be page aligned!
4409	* @param cb The size of the range (in bytes).
4410	* Must be page aligned!
4411	* @param pvBinary Pointer to the binary data backing the ROM image.
4412	* @param cbBinary The size of the binary data pvBinary points to.
4413	* This must be less or equal to @a cb.
4414	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
4415	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
4416	* @param pszDesc Pointer to description string. This must not be freed.
4417	*
4418	* @remark There is no way to remove the rom, automatically on device cleanup or
4419	* manually from the device yet. This isn't difficult in any way, it's
4420	* just not something we expect to be necessary for a while.
4421	*/
4422	VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
4423	const void pvBinary, uint32_t cbBinary, uint32_t fFlags, const char pszDesc)
4424	{
4425	Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p cbBinary=%#x fFlags=%#x pszDesc=%s\n",
4426	pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, cbBinary, fFlags, pszDesc));
4427	pgmLock(pVM);
4428	int rc = pgmR3PhysRomRegisterLocked(pVM, pDevIns, GCPhys, cb, pvBinary, cbBinary, fFlags, pszDesc);
4429	pgmUnlock(pVM);
4430	return rc;
4431	}
4432
4433
4434	/**
4435	* Called by PGMR3MemSetup to reset the shadow, switch to the virgin, and verify
4436	* that the virgin part is untouched.
4437	*
4438	* This is done after the normal memory has been cleared.
4439	*
4440	* ASSUMES that the caller owns the PGM lock.
4441	*
4442	* @param pVM The cross context VM structure.
4443	*/
4444	int pgmR3PhysRomReset(PVM pVM)
4445	{
4446	PGM_LOCK_ASSERT_OWNER(pVM);
4447	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
4448	{
4449	const uint32_t cPages = pRom->cb >> PAGE_SHIFT;
4450
4451	if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
4452	{
4453	/*
4454	* Reset the physical handler.
4455	*/
4456	int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
4457	AssertRCReturn(rc, rc);
4458
4459	/*
4460	* What we do with the shadow pages depends on the memory
4461	* preallocation option. If not enabled, we'll just throw
4462	* out all the dirty pages and replace them by the zero page.
4463	*/
4464	if (!pVM->pgm.s.fRamPreAlloc)
4465	{
4466	/* Free the dirty pages. */
4467	uint32_t cPendingPages = 0;
4468	PGMMFREEPAGESREQ pReq;
4469	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
4470	AssertRCReturn(rc, rc);
4471
4472	for (uint32_t iPage = 0; iPage < cPages; iPage++)
4473	if ( !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
4474	&& !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
4475	{
4476	Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
4477	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
4478	pRom->GCPhys + (iPage << PAGE_SHIFT),
4479	(PGMPAGETYPE)PGM_PAGE_GET_TYPE(&pRom->aPages[iPage].Shadow));
4480	AssertLogRelRCReturn(rc, rc);
4481	}
4482
4483	if (cPendingPages)
4484	{
4485	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
4486	AssertLogRelRCReturn(rc, rc);
4487	}
4488	GMMR3FreePagesCleanup(pReq);
4489	}
4490	else
4491	{
4492	/* clear all the shadow pages. */
4493	for (uint32_t iPage = 0; iPage < cPages; iPage++)
4494	{
4495	if (PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow))
4496	continue;
4497	Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
4498	void *pvDstPage;
4499	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
4500	rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
4501	if (RT_FAILURE(rc))
4502	break;
4503	ASMMemZeroPage(pvDstPage);
4504	}
4505	AssertRCReturn(rc, rc);
4506	}
4507	}
4508
4509	/*
4510	* Restore the original ROM pages after a saved state load.
4511	* Also, in strict builds check that ROM pages remain unmodified.
4512	*/
4513	#ifndef VBOX_STRICT
4514	if (pVM->pgm.s.fRestoreRomPagesOnReset)
4515	#endif
4516	{
4517	size_t cbSrcLeft = pRom->cbOriginal;
4518	uint8_t const pbSrcPage = (uint8_t const )pRom->pvOriginal;
4519	uint32_t cRestored = 0;
4520	for (uint32_t iPage = 0; iPage < cPages && cbSrcLeft > 0; iPage++, pbSrcPage += PAGE_SIZE)
4521	{
4522	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
4523	void const *pvDstPage;
4524	int rc = pgmPhysPageMapReadOnly(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPage);
4525	if (RT_FAILURE(rc))
4526	break;
4527
4528	if (memcmp(pvDstPage, pbSrcPage, RT_MIN(cbSrcLeft, PAGE_SIZE)))
4529	{
4530	if (pVM->pgm.s.fRestoreRomPagesOnReset)
4531	{
4532	void *pvDstPageW;
4533	rc = pgmPhysPageMap(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPageW);
4534	AssertLogRelRCReturn(rc, rc);
4535	memcpy(pvDstPageW, pbSrcPage, RT_MIN(cbSrcLeft, PAGE_SIZE));
4536	cRestored++;
4537	}
4538	else
4539	LogRel(("pgmR3PhysRomReset: %RGp: ROM page changed (%s)\n", GCPhys, pRom->pszDesc));
4540	}
4541	cbSrcLeft -= RT_MIN(cbSrcLeft, PAGE_SIZE);
4542	}
4543	if (cRestored > 0)
4544	LogRel(("PGM: ROM \"%s\": Reloaded %u of %u pages.\n", pRom->pszDesc, cRestored, cPages));
4545	}
4546	}
4547
4548	/* Clear the ROM restore flag now as we only need to do this once after
4549	loading saved state. */
4550	pVM->pgm.s.fRestoreRomPagesOnReset = false;
4551
4552	return VINF_SUCCESS;
4553	}
4554
4555
4556	/**
4557	* Called by PGMR3Term to free resources.
4558	*
4559	* ASSUMES that the caller owns the PGM lock.
4560	*
4561	* @param pVM The cross context VM structure.
4562	*/
4563	void pgmR3PhysRomTerm(PVM pVM)
4564	{
4565	/*
4566	* Free the heap copy of the original bits.
4567	*/
4568	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
4569	{
4570	if ( pRom->pvOriginal
4571	&& !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
4572	{
4573	RTMemFree((void *)pRom->pvOriginal);
4574	pRom->pvOriginal = NULL;
4575	}
4576	}
4577	}
4578
4579
4580	/**
4581	* Change the shadowing of a range of ROM pages.
4582	*
4583	* This is intended for implementing chipset specific memory registers
4584	* and will not be very strict about the input. It will silently ignore
4585	* any pages that are not the part of a shadowed ROM.
4586	*
4587	* @returns VBox status code.
4588	* @retval VINF_PGM_SYNC_CR3
4589	*
4590	* @param pVM The cross context VM structure.
4591	* @param GCPhys Where to start. Page aligned.
4592	* @param cb How much to change. Page aligned.
4593	* @param enmProt The new ROM protection.
4594	*/
4595	VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
4596	{
4597	/*
4598	* Check input
4599	*/
4600	if (!cb)
4601	return VINF_SUCCESS;
4602	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
4603	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
4604	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
4605	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
4606	AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
4607
4608	/*
4609	* Process the request.
4610	*/
4611	pgmLock(pVM);
4612	int rc = VINF_SUCCESS;
4613	bool fFlushTLB = false;
4614	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
4615	{
4616	if ( GCPhys <= pRom->GCPhysLast
4617	&& GCPhysLast >= pRom->GCPhys
4618	&& (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
4619	{
4620	/*
4621	* Iterate the relevant pages and make necessary the changes.
4622	*/
4623	bool fChanges = false;
4624	uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
4625	? pRom->cb >> PAGE_SHIFT
4626	: (GCPhysLast - pRom->GCPhys + 1) >> PAGE_SHIFT;
4627	for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
4628	iPage < cPages;
4629	iPage++)
4630	{
4631	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
4632	if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
4633	{
4634	fChanges = true;
4635
4636	/* flush references to the page. */
4637	PPGMPAGE pRamPage = pgmPhysGetPage(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT));
4638	int rc2 = pgmPoolTrackUpdateGCPhys(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT), pRamPage,
4639	true /fFlushPTEs/, &fFlushTLB);
4640	if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS \|\| RT_FAILURE(rc2)))
4641	rc = rc2;
4642	uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pRamPage);
4643
4644	PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
4645	PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
4646
4647	pOld = pRamPage;
4648	pRamPage = pNew;
4649	/** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
4650
4651	/* Tell NEM about the backing and protection change. */
4652	if (VM_IS_NEM_ENABLED(pVM))
4653	{
4654	PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pNew);
4655	NEMHCNotifyPhysPageChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pOld), PGM_PAGE_GET_HCPHYS(pNew),
4656	pgmPhysPageCalcNemProtection(pRamPage, enmType), enmType, &u2State);
4657	PGM_PAGE_SET_NEM_STATE(pRamPage, u2State);
4658	}
4659	}
4660	pRomPage->enmProt = enmProt;
4661	}
4662
4663	/*
4664	* Reset the access handler if we made changes, no need
4665	* to optimize this.
4666	*/
4667	if (fChanges)
4668	{
4669	int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
4670	if (RT_FAILURE(rc2))
4671	{
4672	pgmUnlock(pVM);
4673	AssertRC(rc);
4674	return rc2;
4675	}
4676	}
4677
4678	/* Advance - cb isn't updated. */
4679	GCPhys = pRom->GCPhys + (cPages << PAGE_SHIFT);
4680	}
4681	}
4682	pgmUnlock(pVM);
4683	if (fFlushTLB)
4684	PGM_INVL_ALL_VCPU_TLBS(pVM);
4685
4686	return rc;
4687	}
4688
4689
4690	/**
4691	* Sets the Address Gate 20 state.
4692	*
4693	* @param pVCpu The cross context virtual CPU structure.
4694	* @param fEnable True if the gate should be enabled.
4695	* False if the gate should be disabled.
4696	*/
4697	VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
4698	{
4699	LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
4700	if (pVCpu->pgm.s.fA20Enabled != fEnable)
4701	{
4702	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4703	PCCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
4704	if ( CPUMIsGuestInVmxRootMode(pCtx)
4705	&& !fEnable)
4706	{
4707	Log(("Cannot enter A20M mode while in VMX root mode\n"));
4708	return;
4709	}
4710	#endif
4711	pVCpu->pgm.s.fA20Enabled = fEnable;
4712	pVCpu->pgm.s.GCPhysA20Mask = ~((RTGCPHYS)!fEnable << 20);
4713	#ifdef VBOX_WITH_REM
4714	REMR3A20Set(pVCpu->pVMR3, pVCpu, fEnable);
4715	#endif
4716	NEMR3NotifySetA20(pVCpu, fEnable);
4717	#ifdef PGM_WITH_A20
4718	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
4719	pgmR3RefreshShadowModeAfterA20Change(pVCpu);
4720	HMFlushTlb(pVCpu);
4721	#endif
4722	IEMTlbInvalidateAllPhysical(pVCpu);
4723	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cA20Changes);
4724	}
4725	}
4726
4727
4728	/**
4729	* Tree enumeration callback for dealing with age rollover.
4730	* It will perform a simple compression of the current age.
4731	*/
4732	static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
4733	{
4734	/* Age compression - ASSUMES iNow == 4. */
4735	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
4736	if (pChunk->iLastUsed >= UINT32_C(0xffffff00))
4737	pChunk->iLastUsed = 3;
4738	else if (pChunk->iLastUsed >= UINT32_C(0xfffff000))
4739	pChunk->iLastUsed = 2;
4740	else if (pChunk->iLastUsed)
4741	pChunk->iLastUsed = 1;
4742	else /* iLastUsed = 0 */
4743	pChunk->iLastUsed = 4;
4744
4745	NOREF(pvUser);
4746	return 0;
4747	}
4748
4749
4750	/**
4751	* The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
4752	*/
4753	typedef struct PGMR3PHYSCHUNKUNMAPCB
4754	{
4755	PVM pVM; /*< Pointer to the VM. /
4756	PPGMCHUNKR3MAP pChunk; /*< The chunk to unmap. /
4757	} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
4758
4759
4760	/**
4761	* Callback used to find the mapping that's been unused for
4762	* the longest time.
4763	*/
4764	static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
4765	{
4766	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
4767	PPGMR3PHYSCHUNKUNMAPCB pArg = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
4768
4769	/*
4770	* Check for locks and compare when last used.
4771	*/
4772	if (pChunk->cRefs)
4773	return 0;
4774	if (pChunk->cPermRefs)
4775	return 0;
4776	if ( pArg->pChunk
4777	&& pChunk->iLastUsed >= pArg->pChunk->iLastUsed)
4778	return 0;
4779
4780	/*
4781	* Check that it's not in any of the TLBs.
4782	*/
4783	PVM pVM = pArg->pVM;
4784	if ( pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(pChunk->Core.Key)].idChunk
4785	== pChunk->Core.Key)
4786	{
4787	pChunk = NULL;
4788	return 0;
4789	}
4790	#ifdef VBOX_STRICT
4791	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
4792	{
4793	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk != pChunk);
4794	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk != pChunk->Core.Key);
4795	}
4796	#endif
4797
4798	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR0.aEntries); i++)
4799	if (pVM->pgm.s.PhysTlbR0.aEntries[i].pMap == pChunk)
4800	return 0;
4801	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR3.aEntries); i++)
4802	if (pVM->pgm.s.PhysTlbR3.aEntries[i].pMap == pChunk)
4803	return 0;
4804
4805	pArg->pChunk = pChunk;
4806	return 0;
4807	}
4808
4809
4810	/**
4811	* Finds a good candidate for unmapping when the ring-3 mapping cache is full.
4812	*
4813	* The candidate will not be part of any TLBs, so no need to flush
4814	* anything afterwards.
4815	*
4816	* @returns Chunk id.
4817	* @param pVM The cross context VM structure.
4818	*/
4819	static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
4820	{
4821	PGM_LOCK_ASSERT_OWNER(pVM);
4822
4823	/*
4824	* Enumerate the age tree starting with the left most node.
4825	*/
4826	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
4827	PGMR3PHYSCHUNKUNMAPCB Args;
4828	Args.pVM = pVM;
4829	Args.pChunk = NULL;
4830	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
4831	Assert(Args.pChunk);
4832	if (Args.pChunk)
4833	{
4834	Assert(Args.pChunk->cRefs == 0);
4835	Assert(Args.pChunk->cPermRefs == 0);
4836	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
4837	return Args.pChunk->Core.Key;
4838	}
4839
4840	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
4841	return INT32_MAX;
4842	}
4843
4844
4845	/**
4846	* Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
4847	*
4848	* This is only called on one of the EMTs while the other ones are waiting for
4849	* it to complete this function.
4850	*
4851	* @returns VINF_SUCCESS (VBox strict status code).
4852	* @param pVM The cross context VM structure.
4853	* @param pVCpu The cross context virtual CPU structure of the calling EMT. Unused.
4854	* @param pvUser User pointer. Unused
4855	*
4856	*/
4857	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
4858	{
4859	int rc = VINF_SUCCESS;
4860	pgmLock(pVM);
4861	NOREF(pVCpu); NOREF(pvUser);
4862
4863	if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
4864	{
4865	/* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
4866	/** @todo also not really efficient to unmap a chunk that contains PD
4867	* or PT pages. */
4868	pgmR3PoolClearAllRendezvous(pVM, &pVM->aCpus[0], NULL /* no need to flush the REM TLB as we already did that above */);
4869
4870	/*
4871	* Request the ring-0 part to unmap a chunk to make space in the mapping cache.
4872	*/
4873	GMMMAPUNMAPCHUNKREQ Req;
4874	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
4875	Req.Hdr.cbReq = sizeof(Req);
4876	Req.pvR3 = NULL;
4877	Req.idChunkMap = NIL_GMM_CHUNKID;
4878	Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
4879	if (Req.idChunkUnmap != INT32_MAX)
4880	{
4881	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
4882	rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
4883	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
4884	if (RT_SUCCESS(rc))
4885	{
4886	/*
4887	* Remove the unmapped one.
4888	*/
4889	PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
4890	AssertRelease(pUnmappedChunk);
4891	AssertRelease(!pUnmappedChunk->cRefs);
4892	AssertRelease(!pUnmappedChunk->cPermRefs);
4893	pUnmappedChunk->pv = NULL;
4894	pUnmappedChunk->Core.Key = UINT32_MAX;
4895	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
4896	MMR3HeapFree(pUnmappedChunk);
4897	#else
4898	MMR3UkHeapFree(pVM, pUnmappedChunk, MM_TAG_PGM_CHUNK_MAPPING);
4899	#endif
4900	pVM->pgm.s.ChunkR3Map.c--;
4901	pVM->pgm.s.cUnmappedChunks++;
4902
4903	/*
4904	* Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses).
4905	*/
4906	/** @todo We should not flush chunks which include cr3 mappings. */
4907	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4908	{
4909	PPGMCPU pPGM = &pVM->aCpus[idCpu].pgm.s;
4910
4911	pPGM->pGst32BitPdR3 = NULL;
4912	pPGM->pGstPaePdptR3 = NULL;
4913	pPGM->pGstAmd64Pml4R3 = NULL;
4914	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4915	pPGM->pGst32BitPdR0 = NIL_RTR0PTR;
4916	pPGM->pGstPaePdptR0 = NIL_RTR0PTR;
4917	pPGM->pGstAmd64Pml4R0 = NIL_RTR0PTR;
4918	#endif
4919	for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
4920	{
4921	pPGM->apGstPaePDsR3[i] = NULL;
4922	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4923	pPGM->apGstPaePDsR0[i] = NIL_RTR0PTR;
4924	#endif
4925	}
4926
4927	/* Flush REM TLBs. */
4928	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
4929	}
4930	#ifdef VBOX_WITH_REM
4931	/* Flush REM translation blocks. */
4932	REMFlushTBs(pVM);
4933	#endif
4934	}
4935	}
4936	}
4937	pgmUnlock(pVM);
4938	return rc;
4939	}
4940
4941	/**
4942	* Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
4943	*
4944	* @returns VBox status code.
4945	* @param pVM The cross context VM structure.
4946	*/
4947	void pgmR3PhysUnmapChunk(PVM pVM)
4948	{
4949	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
4950	AssertRC(rc);
4951	}
4952
4953
4954	/**
4955	* Maps the given chunk into the ring-3 mapping cache.
4956	*
4957	* This will call ring-0.
4958	*
4959	* @returns VBox status code.
4960	* @param pVM The cross context VM structure.
4961	* @param idChunk The chunk in question.
4962	* @param ppChunk Where to store the chunk tracking structure.
4963	*
4964	* @remarks Called from within the PGM critical section.
4965	* @remarks Can be called from any thread!
4966	*/
4967	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
4968	{
4969	int rc;
4970
4971	PGM_LOCK_ASSERT_OWNER(pVM);
4972
4973	/*
4974	* Move the chunk time forward.
4975	*/
4976	pVM->pgm.s.ChunkR3Map.iNow++;
4977	if (pVM->pgm.s.ChunkR3Map.iNow == 0)
4978	{
4979	pVM->pgm.s.ChunkR3Map.iNow = 4;
4980	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkAgeingRolloverCallback, NULL);
4981	}
4982
4983	/*
4984	* Allocate a new tracking structure first.
4985	*/
4986	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
4987	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
4988	#else
4989	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3UkHeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk), NULL);
4990	#endif
4991	AssertReturn(pChunk, VERR_NO_MEMORY);
4992	pChunk->Core.Key = idChunk;
4993	pChunk->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
4994
4995	/*
4996	* Request the ring-0 part to map the chunk in question.
4997	*/
4998	GMMMAPUNMAPCHUNKREQ Req;
4999	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
5000	Req.Hdr.cbReq = sizeof(Req);
5001	Req.pvR3 = NULL;
5002	Req.idChunkMap = idChunk;
5003	Req.idChunkUnmap = NIL_GMM_CHUNKID;
5004
5005	/* Must be callable from any thread, so can't use VMMR3CallR0. */
5006	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
5007	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
5008	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
5009	if (RT_SUCCESS(rc))
5010	{
5011	pChunk->pv = Req.pvR3;
5012
5013	/*
5014	* If we're running out of virtual address space, then we should
5015	* unmap another chunk.
5016	*
5017	* Currently, an unmap operation requires that all other virtual CPUs
5018	* are idling and not by chance making use of the memory we're
5019	* unmapping. So, we create an async unmap operation here.
5020	*
5021	* Now, when creating or restoring a saved state this wont work very
5022	* well since we may want to restore all guest RAM + a little something.
5023	* So, we have to do the unmap synchronously. Fortunately for us
5024	* though, during these operations the other virtual CPUs are inactive
5025	* and it should be safe to do this.
5026	*/
5027	/** @todo Eventually we should lock all memory when used and do
5028	* map+unmap as one kernel call without any rendezvous or
5029	* other precautions. */
5030	if (pVM->pgm.s.ChunkR3Map.c + 1 >= pVM->pgm.s.ChunkR3Map.cMax)
5031	{
5032	switch (VMR3GetState(pVM))
5033	{
5034	case VMSTATE_LOADING:
5035	case VMSTATE_SAVING:
5036	{
5037	PVMCPU pVCpu = VMMGetCpu(pVM);
5038	if ( pVCpu
5039	&& pVM->pgm.s.cDeprecatedPageLocks == 0)
5040	{
5041	pgmR3PhysUnmapChunkRendezvous(pVM, pVCpu, NULL);
5042	break;
5043	}
5044	}
5045	RT_FALL_THRU();
5046	default:
5047	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
5048	AssertRC(rc);
5049	break;
5050	}
5051	}
5052
5053	/*
5054	* Update the tree. We must do this after any unmapping to make sure
5055	* the chunk we're going to return isn't unmapped by accident.
5056	*/
5057	AssertPtr(Req.pvR3);
5058	bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
5059	AssertRelease(fRc);
5060	pVM->pgm.s.ChunkR3Map.c++;
5061	pVM->pgm.s.cMappedChunks++;
5062	}
5063	else
5064	{
5065	/** @todo this may fail because of /proc/sys/vm/max_map_count, so we
5066	* should probably restrict ourselves on linux. */
5067	AssertRC(rc);
5068	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
5069	MMR3HeapFree(pChunk);
5070	#else
5071	MMR3UkHeapFree(pVM, pChunk, MM_TAG_PGM_CHUNK_MAPPING);
5072	#endif
5073	pChunk = NULL;
5074	}
5075
5076	*ppChunk = pChunk;
5077	return rc;
5078	}
5079
5080
5081	/**
5082	* For VMMCALLRING3_PGM_MAP_CHUNK, considered internal.
5083	*
5084	* @returns see pgmR3PhysChunkMap.
5085	* @param pVM The cross context VM structure.
5086	* @param idChunk The chunk to map.
5087	*/
5088	VMMR3DECL(int) PGMR3PhysChunkMap(PVM pVM, uint32_t idChunk)
5089	{
5090	PPGMCHUNKR3MAP pChunk;
5091	int rc;
5092
5093	pgmLock(pVM);
5094	rc = pgmR3PhysChunkMap(pVM, idChunk, &pChunk);
5095	pgmUnlock(pVM);
5096	return rc;
5097	}
5098
5099
5100	/**
5101	* Invalidates the TLB for the ring-3 mapping cache.
5102	*
5103	* @param pVM The cross context VM structure.
5104	*/
5105	VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
5106	{
5107	pgmLock(pVM);
5108	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
5109	{
5110	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
5111	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
5112	}
5113	/* The page map TLB references chunks, so invalidate that one too. */
5114	pgmPhysInvalidatePageMapTLB(pVM);
5115	pgmUnlock(pVM);
5116	}
5117
5118
5119	/**
5120	* Response to VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE to allocate a large
5121	* (2MB) page for use with a nested paging PDE.
5122	*
5123	* @returns The following VBox status codes.
5124	* @retval VINF_SUCCESS on success.
5125	* @retval VINF_EM_NO_MEMORY if we're out of memory.
5126	*
5127	* @param pVM The cross context VM structure.
5128	* @param GCPhys GC physical start address of the 2 MB range
5129	*/
5130	VMMR3DECL(int) PGMR3PhysAllocateLargeHandyPage(PVM pVM, RTGCPHYS GCPhys)
5131	{
5132	#ifdef PGM_WITH_LARGE_PAGES
5133	uint64_t u64TimeStamp1, u64TimeStamp2;
5134
5135	pgmLock(pVM);
5136
5137	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
5138	u64TimeStamp1 = RTTimeMilliTS();
5139	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE, 0, NULL);
5140	u64TimeStamp2 = RTTimeMilliTS();
5141	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
5142	if (RT_SUCCESS(rc))
5143	{
5144	Assert(pVM->pgm.s.cLargeHandyPages == 1);
5145
5146	uint32_t idPage = pVM->pgm.s.aLargeHandyPage[0].idPage;
5147	RTHCPHYS HCPhys = pVM->pgm.s.aLargeHandyPage[0].HCPhysGCPhys;
5148
5149	void *pv;
5150
5151	/* Map the large page into our address space.
5152	*
5153	* Note: assuming that within the 2 MB range:
5154	* - GCPhys + PAGE_SIZE = HCPhys + PAGE_SIZE (whole point of this exercise)
5155	* - user space mapping is continuous as well
5156	* - page id (GCPhys) + 1 = page id (GCPhys + PAGE_SIZE)
5157	*/
5158	rc = pgmPhysPageMapByPageID(pVM, idPage, HCPhys, &pv);
5159	AssertLogRelMsg(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", idPage, HCPhys, rc));
5160
5161	if (RT_SUCCESS(rc))
5162	{
5163	/*
5164	* Clear the pages.
5165	*/
5166	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
5167	for (unsigned i = 0; i < _2M/PAGE_SIZE; i++)
5168	{
5169	ASMMemZeroPage(pv);
5170
5171	PPGMPAGE pPage;
5172	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
5173	AssertRC(rc);
5174
5175	Assert(PGM_PAGE_IS_ZERO(pPage));
5176	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
5177	pVM->pgm.s.cZeroPages--;
5178
5179	/*
5180	* Do the PGMPAGE modifications.
5181	*/
5182	pVM->pgm.s.cPrivatePages++;
5183	PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
5184	PGM_PAGE_SET_PAGEID(pVM, pPage, idPage);
5185	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
5186	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE);
5187	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
5188	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
5189
5190	/* Somewhat dirty assumption that page ids are increasing. */
5191	idPage++;
5192
5193	HCPhys += PAGE_SIZE;
5194	GCPhys += PAGE_SIZE;
5195
5196	pv = (void *)((uintptr_t)pv + PAGE_SIZE);
5197
5198	Log3(("PGMR3PhysAllocateLargePage: idPage=%#x HCPhys=%RGp\n", idPage, HCPhys));
5199	}
5200	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
5201
5202	/* Flush all TLBs. */
5203	PGM_INVL_ALL_VCPU_TLBS(pVM);
5204	pgmPhysInvalidatePageMapTLB(pVM);
5205	}
5206	pVM->pgm.s.cLargeHandyPages = 0;
5207	}
5208
5209	if (RT_SUCCESS(rc))
5210	{
5211	static uint32_t cTimeOut = 0;
5212	uint64_t u64TimeStampDelta = u64TimeStamp2 - u64TimeStamp1;
5213
5214	if (u64TimeStampDelta > 100)
5215	{
5216	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatLargePageOverflow);
5217	if ( ++cTimeOut > 10
5218	\|\| u64TimeStampDelta > 1000 /* more than one second forces an early retirement from allocating large pages. */)
5219	{
5220	/* If repeated attempts to allocate a large page takes more than 100 ms, then we fall back to normal 4k pages.
5221	* E.g. Vista 64 tries to move memory around, which takes a huge amount of time.
5222	*/
5223	LogRel(("PGMR3PhysAllocateLargePage: allocating large pages takes too long (last attempt %d ms; nr of timeouts %d); DISABLE\n", u64TimeStampDelta, cTimeOut));
5224	PGMSetLargePageUsage(pVM, false);
5225	}
5226	}
5227	else
5228	if (cTimeOut > 0)
5229	cTimeOut--;
5230	}
5231
5232	pgmUnlock(pVM);
5233	return rc;
5234	#else
5235	RT_NOREF(pVM, GCPhys);
5236	return VERR_NOT_IMPLEMENTED;
5237	#endif /* PGM_WITH_LARGE_PAGES */
5238	}
5239
5240
5241	/**
5242	* Response to VM_FF_PGM_NEED_HANDY_PAGES and VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES.
5243	*
5244	* This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
5245	* signal and clear the out of memory condition. When contracted, this API is
5246	* used to try clear the condition when the user wants to resume.
5247	*
5248	* @returns The following VBox status codes.
5249	* @retval VINF_SUCCESS on success. FFs cleared.
5250	* @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
5251	* this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
5252	*
5253	* @param pVM The cross context VM structure.
5254	*
5255	* @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
5256	* in EM.cpp and shouldn't be propagated outside TRPM, HM, EM and
5257	* pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
5258	* handler.
5259	*/
5260	VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
5261	{
5262	pgmLock(pVM);
5263
5264	/*
5265	* Allocate more pages, noting down the index of the first new page.
5266	*/
5267	uint32_t iClear = pVM->pgm.s.cHandyPages;
5268	AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_PGM_HANDY_PAGE_IPE);
5269	Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
5270	int rcAlloc = VINF_SUCCESS;
5271	int rcSeed = VINF_SUCCESS;
5272	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
5273	while (rc == VERR_GMM_SEED_ME)
5274	{
5275	void *pvChunk;
5276	rcAlloc = rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
5277	if (RT_SUCCESS(rc))
5278	{
5279	rcSeed = rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
5280	if (RT_FAILURE(rc))
5281	SUPR3PageFree(pvChunk, GMM_CHUNK_SIZE >> PAGE_SHIFT);
5282	}
5283	if (RT_SUCCESS(rc))
5284	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
5285	}
5286
5287	/** @todo we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
5288	if ( rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
5289	&& pVM->pgm.s.cHandyPages > 0)
5290	{
5291	/* Still handy pages left, so don't panic. */
5292	rc = VINF_SUCCESS;
5293	}
5294
5295	if (RT_SUCCESS(rc))
5296	{
5297	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
5298	Assert(pVM->pgm.s.cHandyPages > 0);
5299	VM_FF_CLEAR(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
5300	VM_FF_CLEAR(pVM, VM_FF_PGM_NO_MEMORY);
5301
5302	#ifdef VBOX_STRICT
5303	uint32_t i;
5304	for (i = iClear; i < pVM->pgm.s.cHandyPages; i++)
5305	if ( pVM->pgm.s.aHandyPages[i].idPage == NIL_GMM_PAGEID
5306	\|\| pVM->pgm.s.aHandyPages[i].idSharedPage != NIL_GMM_PAGEID
5307	\|\| (pVM->pgm.s.aHandyPages[i].HCPhysGCPhys & PAGE_OFFSET_MASK))
5308	break;
5309	if (i != pVM->pgm.s.cHandyPages)
5310	{
5311	RTAssertMsg1Weak(NULL, __LINE__, __FILE__, __FUNCTION__);
5312	RTAssertMsg2Weak("i=%d iClear=%d cHandyPages=%d\n", i, iClear, pVM->pgm.s.cHandyPages);
5313	for (uint32_t j = iClear; j < pVM->pgm.s.cHandyPages; j++)
5314	RTAssertMsg2Add("%03d: idPage=%d HCPhysGCPhys=%RHp idSharedPage=%d%\n", j,
5315	pVM->pgm.s.aHandyPages[j].idPage,
5316	pVM->pgm.s.aHandyPages[j].HCPhysGCPhys,
5317	pVM->pgm.s.aHandyPages[j].idSharedPage,
5318	j == i ? " <---" : "");
5319	RTAssertPanic();
5320	}
5321	#endif
5322	/*
5323	* Clear the pages.
5324	*/
5325	while (iClear < pVM->pgm.s.cHandyPages)
5326	{
5327	PGMMPAGEDESC pPage = &pVM->pgm.s.aHandyPages[iClear];
5328	void *pv;
5329	rc = pgmPhysPageMapByPageID(pVM, pPage->idPage, pPage->HCPhysGCPhys, &pv);
5330	AssertLogRelMsgBreak(RT_SUCCESS(rc),
5331	("%u/%u: idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n",
5332	iClear, pVM->pgm.s.cHandyPages, pPage->idPage, pPage->HCPhysGCPhys, rc));
5333	ASMMemZeroPage(pv);
5334	iClear++;
5335	Log3(("PGMR3PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
5336	}
5337	}
5338	else
5339	{
5340	uint64_t cAllocPages, cMaxPages, cBalloonPages;
5341
5342	/*
5343	* We should never get here unless there is a genuine shortage of
5344	* memory (or some internal error). Flag the error so the VM can be
5345	* suspended ASAP and the user informed. If we're totally out of
5346	* handy pages we will return failure.
5347	*/
5348	/* Report the failure. */
5349	LogRel(("PGM: Failed to procure handy pages; rc=%Rrc rcAlloc=%Rrc rcSeed=%Rrc cHandyPages=%#x\n"
5350	" cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
5351	rc, rcAlloc, rcSeed,
5352	pVM->pgm.s.cHandyPages,
5353	pVM->pgm.s.cAllPages,
5354	pVM->pgm.s.cPrivatePages,
5355	pVM->pgm.s.cSharedPages,
5356	pVM->pgm.s.cZeroPages));
5357
5358	if (GMMR3QueryMemoryStats(pVM, &cAllocPages, &cMaxPages, &cBalloonPages) == VINF_SUCCESS)
5359	{
5360	LogRel(("GMM: Statistics:\n"
5361	" Allocated pages: %RX64\n"
5362	" Maximum pages: %RX64\n"
5363	" Ballooned pages: %RX64\n", cAllocPages, cMaxPages, cBalloonPages));
5364	}
5365
5366	if ( rc != VERR_NO_MEMORY
5367	&& rc != VERR_NO_PHYS_MEMORY
5368	&& rc != VERR_LOCK_FAILED)
5369	{
5370	for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
5371	{
5372	LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
5373	i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
5374	pVM->pgm.s.aHandyPages[i].idSharedPage));
5375	uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
5376	if (idPage != NIL_GMM_PAGEID)
5377	{
5378	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
5379	pRam;
5380	pRam = pRam->pNextR3)
5381	{
5382	uint32_t const cPages = pRam->cb >> PAGE_SHIFT;
5383	for (uint32_t iPage = 0; iPage < cPages; iPage++)
5384	if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
5385	LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
5386	pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
5387	}
5388	}
5389	}
5390	}
5391
5392	if (rc == VERR_NO_MEMORY)
5393	{
5394	uint64_t cbHostRamAvail = 0;
5395	int rc2 = RTSystemQueryAvailableRam(&cbHostRamAvail);
5396	if (RT_SUCCESS(rc2))
5397	LogRel(("Host RAM: %RU64MB available\n", cbHostRamAvail / _1M));
5398	else
5399	LogRel(("Cannot determine the amount of available host memory\n"));
5400	}
5401
5402	/* Set the FFs and adjust rc. */
5403	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
5404	VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
5405	if ( rc == VERR_NO_MEMORY
5406	\|\| rc == VERR_NO_PHYS_MEMORY
5407	\|\| rc == VERR_LOCK_FAILED)
5408	rc = VINF_EM_NO_MEMORY;
5409	}
5410
5411	pgmUnlock(pVM);
5412	return rc;
5413	}
5414
5415
5416	/**
5417	* Frees the specified RAM page and replaces it with the ZERO page.
5418	*
5419	* This is used by ballooning, remapping MMIO2, RAM reset and state loading.
5420	*
5421	* @param pVM The cross context VM structure.
5422	* @param pReq Pointer to the request.
5423	* @param pcPendingPages Where the number of pages waiting to be freed are
5424	* kept. This will normally be incremented.
5425	* @param pPage Pointer to the page structure.
5426	* @param GCPhys The guest physical address of the page, if applicable.
5427	* @param enmNewType New page type for NEM notification, since several
5428	* callers will change the type upon successful return.
5429	*
5430	* @remarks The caller must own the PGM lock.
5431	*/
5432	int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys,
5433	PGMPAGETYPE enmNewType)
5434	{
5435	/*
5436	* Assert sanity.
5437	*/
5438	PGM_LOCK_ASSERT_OWNER(pVM);
5439	if (RT_UNLIKELY( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
5440	&& PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
5441	{
5442	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
5443	return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
5444	}
5445
5446	/** @todo What about ballooning of large pages??! */
5447	Assert( PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
5448	&& PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE_DISABLED);
5449
5450	if ( PGM_PAGE_IS_ZERO(pPage)
5451	\|\| PGM_PAGE_IS_BALLOONED(pPage))
5452	return VINF_SUCCESS;
5453
5454	const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
5455	Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
5456	if (RT_UNLIKELY( idPage == NIL_GMM_PAGEID
5457	\|\| idPage > GMM_PAGEID_LAST
5458	\|\| PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID))
5459	{
5460	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
5461	return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
5462	}
5463	const RTHCPHYS HCPhysPrev = PGM_PAGE_GET_HCPHYS(pPage);
5464
5465	/* update page count stats. */
5466	if (PGM_PAGE_IS_SHARED(pPage))
5467	pVM->pgm.s.cSharedPages--;
5468	else
5469	pVM->pgm.s.cPrivatePages--;
5470	pVM->pgm.s.cZeroPages++;
5471
5472	/* Deal with write monitored pages. */
5473	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
5474	{
5475	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
5476	pVM->pgm.s.cWrittenToPages++;
5477	}
5478
5479	/*
5480	* pPage = ZERO page.
5481	*/
5482	PGM_PAGE_SET_HCPHYS(pVM, pPage, pVM->pgm.s.HCPhysZeroPg);
5483	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
5484	PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
5485	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
5486	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
5487	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
5488
5489	/* Flush physical page map TLB entry. */
5490	pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
5491
5492	/* Notify NEM. */
5493	/** @todo consider doing batch NEM notifications. */
5494	if (VM_IS_NEM_ENABLED(pVM))
5495	{
5496	uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
5497	NEMHCNotifyPhysPageChanged(pVM, GCPhys, HCPhysPrev, pVM->pgm.s.HCPhysZeroPg,
5498	pgmPhysPageCalcNemProtection(pPage, enmNewType), enmNewType, &u2State);
5499	PGM_PAGE_SET_NEM_STATE(pPage, u2State);
5500	}
5501
5502	/*
5503	* Make sure it's not in the handy page array.
5504	*/
5505	for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
5506	{
5507	if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
5508	{
5509	pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
5510	break;
5511	}
5512	if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
5513	{
5514	pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
5515	break;
5516	}
5517	}
5518
5519	/*
5520	* Push it onto the page array.
5521	*/
5522	uint32_t iPage = *pcPendingPages;
5523	Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
5524	*pcPendingPages += 1;
5525
5526	pReq->aPages[iPage].idPage = idPage;
5527
5528	if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
5529	return VINF_SUCCESS;
5530
5531	/*
5532	* Flush the pages.
5533	*/
5534	int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
5535	if (RT_SUCCESS(rc))
5536	{
5537	GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
5538	*pcPendingPages = 0;
5539	}
5540	return rc;
5541	}
5542
5543
5544	/**
5545	* Converts a GC physical address to a HC ring-3 pointer, with some
5546	* additional checks.
5547	*
5548	* @returns VBox status code.
5549	* @retval VINF_SUCCESS on success.
5550	* @retval VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
5551	* access handler of some kind.
5552	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
5553	* accesses or is odd in any way.
5554	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
5555	*
5556	* @param pVM The cross context VM structure.
5557	* @param GCPhys The GC physical address to convert. Since this is only
5558	* used for filling the REM TLB, the A20 mask must be
5559	* applied before calling this API.
5560	* @param fWritable Whether write access is required.
5561	* @param ppv Where to store the pointer corresponding to GCPhys on
5562	* success.
5563	*/
5564	VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
5565	{
5566	pgmLock(pVM);
5567	PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
5568
5569	PPGMRAMRANGE pRam;
5570	PPGMPAGE pPage;
5571	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
5572	if (RT_SUCCESS(rc))
5573	{
5574	if (PGM_PAGE_IS_BALLOONED(pPage))
5575	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
5576	else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
5577	rc = VINF_SUCCESS;
5578	else
5579	{
5580	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
5581	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
5582	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
5583	{
5584	/** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
5585	* in -norawr0 mode. */
5586	if (fWritable)
5587	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
5588	}
5589	else
5590	{
5591	/* Temporarily disabled physical handler(s), since the recompiler
5592	doesn't get notified when it's reset we'll have to pretend it's
5593	operating normally. */
5594	if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
5595	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
5596	else
5597	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
5598	}
5599	}
5600	if (RT_SUCCESS(rc))
5601	{
5602	int rc2;
5603
5604	/* Make sure what we return is writable. */
5605	if (fWritable)
5606	switch (PGM_PAGE_GET_STATE(pPage))
5607	{
5608	case PGM_PAGE_STATE_ALLOCATED:
5609	break;
5610	case PGM_PAGE_STATE_BALLOONED:
5611	AssertFailed();
5612	break;
5613	case PGM_PAGE_STATE_ZERO:
5614	case PGM_PAGE_STATE_SHARED:
5615	if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
5616	break;
5617	RT_FALL_THRU();
5618	case PGM_PAGE_STATE_WRITE_MONITORED:
5619	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
5620	AssertLogRelRCReturn(rc2, rc2);
5621	break;
5622	}
5623
5624	/* Get a ring-3 mapping of the address. */
5625	PPGMPAGER3MAPTLBE pTlbe;
5626	rc2 = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
5627	AssertLogRelRCReturn(rc2, rc2);
5628	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
5629	/** @todo mapping/locking hell; this isn't horribly efficient since
5630	* pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
5631
5632	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
5633	}
5634	else
5635	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
5636
5637	/* else: handler catching all access, no pointer returned. */
5638	}
5639	else
5640	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
5641
5642	pgmUnlock(pVM);
5643	return rc;
5644	}
5645

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp@ 80182

Download in other formats: