PGMPhys.cpp@ 23488

Last change on this file since 23488 was 23488, checked in by vboxsync, 15 years ago
PGM: Lock stats and check for write locks on monitored pages during live save.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 117.7 KB

Line
1	/* $Id: PGMPhys.cpp 23488 2009-10-01 15:38:06Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_PGM_PHYS
27	#include <VBox/pgm.h>
28	#include <VBox/iom.h>
29	#include <VBox/mm.h>
30	#include <VBox/stam.h>
31	#include <VBox/rem.h>
32	#include <VBox/pdmdev.h>
33	#include "PGMInternal.h"
34	#include <VBox/vm.h>
35	#include <VBox/sup.h>
36	#include <VBox/param.h>
37	#include <VBox/err.h>
38	#include <VBox/log.h>
39	#include <iprt/assert.h>
40	#include <iprt/alloc.h>
41	#include <iprt/asm.h>
42	#include <iprt/thread.h>
43	#include <iprt/string.h>
44
45
46	/*******************************************************************************
47	* Defined Constants And Macros *
48	*******************************************************************************/
49	/** The number of pages to free in one batch. */
50	#define PGMPHYS_FREE_PAGE_BATCH_SIZE 128
51
52
53	/*******************************************************************************
54	* Internal Functions *
55	*******************************************************************************/
56	static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
57	static int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys);
58
59
60	/*
61	* PGMR3PhysReadU8-64
62	* PGMR3PhysWriteU8-64
63	*/
64	#define PGMPHYSFN_READNAME PGMR3PhysReadU8
65	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
66	#define PGMPHYS_DATASIZE 1
67	#define PGMPHYS_DATATYPE uint8_t
68	#include "PGMPhysRWTmpl.h"
69
70	#define PGMPHYSFN_READNAME PGMR3PhysReadU16
71	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
72	#define PGMPHYS_DATASIZE 2
73	#define PGMPHYS_DATATYPE uint16_t
74	#include "PGMPhysRWTmpl.h"
75
76	#define PGMPHYSFN_READNAME PGMR3PhysReadU32
77	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
78	#define PGMPHYS_DATASIZE 4
79	#define PGMPHYS_DATATYPE uint32_t
80	#include "PGMPhysRWTmpl.h"
81
82	#define PGMPHYSFN_READNAME PGMR3PhysReadU64
83	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
84	#define PGMPHYS_DATASIZE 8
85	#define PGMPHYS_DATATYPE uint64_t
86	#include "PGMPhysRWTmpl.h"
87
88
89	/**
90	* EMT worker for PGMR3PhysReadExternal.
91	*/
92	static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead)
93	{
94	PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead);
95	return VINF_SUCCESS;
96	}
97
98
99	/**
100	* Write to physical memory, external users.
101	*
102	* @returns VBox status code.
103	* @retval VINF_SUCCESS.
104	*
105	* @param pVM VM Handle.
106	* @param GCPhys Physical address to write to.
107	* @param pvBuf What to write.
108	* @param cbWrite How many bytes to write.
109	*
110	* @thread Any but EMTs.
111	*/
112	VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
113	{
114	VM_ASSERT_OTHER_THREAD(pVM);
115
116	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
117	LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
118
119	pgmLock(pVM);
120
121	/*
122	* Copy loop on ram ranges.
123	*/
124	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
125	for (;;)
126	{
127	/* Find range. */
128	while (pRam && GCPhys > pRam->GCPhysLast)
129	pRam = pRam->CTX_SUFF(pNext);
130	/* Inside range or not? */
131	if (pRam && GCPhys >= pRam->GCPhys)
132	{
133	/*
134	* Must work our way thru this page by page.
135	*/
136	RTGCPHYS off = GCPhys - pRam->GCPhys;
137	while (off < pRam->cb)
138	{
139	unsigned iPage = off >> PAGE_SHIFT;
140	PPGMPAGE pPage = &pRam->aPages[iPage];
141
142	/*
143	* If the page has an ALL access handler, we'll have to
144	* delegate the job to EMT.
145	*/
146	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
147	{
148	pgmUnlock(pVM);
149
150	return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 4,
151	pVM, &GCPhys, pvBuf, cbRead);
152	}
153	Assert(!PGM_PAGE_IS_MMIO(pPage));
154
155	/*
156	* Simple stuff, go ahead.
157	*/
158	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
159	if (cb > cbRead)
160	cb = cbRead;
161	const void *pvSrc;
162	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc);
163	if (RT_SUCCESS(rc))
164	memcpy(pvBuf, pvSrc, cb);
165	else
166	{
167	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
168	pRam->GCPhys + off, pPage, rc));
169	memset(pvBuf, 0xff, cb);
170	}
171
172	/* next page */
173	if (cb >= cbRead)
174	{
175	pgmUnlock(pVM);
176	return VINF_SUCCESS;
177	}
178	cbRead -= cb;
179	off += cb;
180	GCPhys += cb;
181	pvBuf = (char *)pvBuf + cb;
182	} /* walk pages in ram range. */
183	}
184	else
185	{
186	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
187
188	/*
189	* Unassigned address space.
190	*/
191	if (!pRam)
192	break;
193	size_t cb = pRam->GCPhys - GCPhys;
194	if (cb >= cbRead)
195	{
196	memset(pvBuf, 0xff, cbRead);
197	break;
198	}
199	memset(pvBuf, 0xff, cb);
200
201	cbRead -= cb;
202	pvBuf = (char *)pvBuf + cb;
203	GCPhys += cb;
204	}
205	} /* Ram range walk */
206
207	pgmUnlock(pVM);
208
209	return VINF_SUCCESS;
210	}
211
212
213	/**
214	* EMT worker for PGMR3PhysWriteExternal.
215	*/
216	static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite)
217	{
218	/** @todo VERR_EM_NO_MEMORY */
219	PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite);
220	return VINF_SUCCESS;
221	}
222
223
224	/**
225	* Write to physical memory, external users.
226	*
227	* @returns VBox status code.
228	* @retval VINF_SUCCESS.
229	* @retval VERR_EM_NO_MEMORY.
230	*
231	* @param pVM VM Handle.
232	* @param GCPhys Physical address to write to.
233	* @param pvBuf What to write.
234	* @param cbWrite How many bytes to write.
235	*
236	* @thread Any but EMTs.
237	*/
238	VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite)
239	{
240	VM_ASSERT_OTHER_THREAD(pVM);
241
242	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites, ("Calling PGMR3PhysWriteExternal after pgmR3Save()!\n"));
243	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
244	LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
245
246	pgmLock(pVM);
247
248	/*
249	* Copy loop on ram ranges, stop when we hit something difficult.
250	*/
251	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
252	for (;;)
253	{
254	/* Find range. */
255	while (pRam && GCPhys > pRam->GCPhysLast)
256	pRam = pRam->CTX_SUFF(pNext);
257	/* Inside range or not? */
258	if (pRam && GCPhys >= pRam->GCPhys)
259	{
260	/*
261	* Must work our way thru this page by page.
262	*/
263	RTGCPTR off = GCPhys - pRam->GCPhys;
264	while (off < pRam->cb)
265	{
266	RTGCPTR iPage = off >> PAGE_SHIFT;
267	PPGMPAGE pPage = &pRam->aPages[iPage];
268
269	/*
270	* It the page is in any way problematic, we have to
271	* do the work on the EMT. Anything that needs to be made
272	* writable or involves access handlers is problematic.
273	*/
274	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
275	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
276	{
277	pgmUnlock(pVM);
278
279	return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 4,
280	pVM, &GCPhys, pvBuf, cbWrite);
281	}
282	Assert(!PGM_PAGE_IS_MMIO(pPage));
283
284	/*
285	* Simple stuff, go ahead.
286	*/
287	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
288	if (cb > cbWrite)
289	cb = cbWrite;
290	void *pvDst;
291	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst);
292	if (RT_SUCCESS(rc))
293	memcpy(pvDst, pvBuf, cb);
294	else
295	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
296	pRam->GCPhys + off, pPage, rc));
297
298	/* next page */
299	if (cb >= cbWrite)
300	{
301	pgmUnlock(pVM);
302	return VINF_SUCCESS;
303	}
304
305	cbWrite -= cb;
306	off += cb;
307	GCPhys += cb;
308	pvBuf = (const char *)pvBuf + cb;
309	} /* walk pages in ram range */
310	}
311	else
312	{
313	/*
314	* Unassigned address space, skip it.
315	*/
316	if (!pRam)
317	break;
318	size_t cb = pRam->GCPhys - GCPhys;
319	if (cb >= cbWrite)
320	break;
321	cbWrite -= cb;
322	pvBuf = (const char *)pvBuf + cb;
323	GCPhys += cb;
324	}
325	} /* Ram range walk */
326
327	pgmUnlock(pVM);
328	return VINF_SUCCESS;
329	}
330
331
332	/**
333	* VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
334	*
335	* @returns see PGMR3PhysGCPhys2CCPtrExternal
336	* @param pVM The VM handle.
337	* @param pGCPhys Pointer to the guest physical address.
338	* @param ppv Where to store the mapping address.
339	* @param pLock Where to store the lock.
340	*/
341	static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
342	{
343	/*
344	* Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
345	* an access handler after it succeeds.
346	*/
347	int rc = pgmLock(pVM);
348	AssertRCReturn(rc, rc);
349
350	rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
351	if (RT_SUCCESS(rc))
352	{
353	PPGMPAGEMAPTLBE pTlbe;
354	int rc2 = pgmPhysPageQueryTlbe(&pVM->pgm.s, *pGCPhys, &pTlbe);
355	AssertFatalRC(rc2);
356	PPGMPAGE pPage = pTlbe->pPage;
357	if (PGM_PAGE_IS_MMIO(pPage))
358	{
359	PGMPhysReleasePageMappingLock(pVM, pLock);
360	rc = VERR_PGM_PHYS_PAGE_RESERVED;
361	}
362	else if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
363	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
364	\|\| pgmPoolIsDirtyPage(pVM, *pGCPhys)
365	#endif
366	)
367	{
368	/* We must flush any corresponding pgm pool page here, otherwise we'll
369	* not be informed about writes and keep bogus gst->shw mappings around.
370	*/
371	pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
372	Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
373	/** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
374	* active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
375	}
376	}
377
378	pgmUnlock(pVM);
379	return rc;
380	}
381
382
383	/**
384	* Requests the mapping of a guest page into ring-3, external threads.
385	*
386	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
387	* release it.
388	*
389	* This API will assume your intention is to write to the page, and will
390	* therefore replace shared and zero pages. If you do not intend to modify the
391	* page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
392	*
393	* @returns VBox status code.
394	* @retval VINF_SUCCESS on success.
395	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
396	* backing or if the page has any active access handlers. The caller
397	* must fall back on using PGMR3PhysWriteExternal.
398	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
399	*
400	* @param pVM The VM handle.
401	* @param GCPhys The guest physical address of the page that should be mapped.
402	* @param ppv Where to store the address corresponding to GCPhys.
403	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
404	*
405	* @remark Avoid calling this API from within critical sections (other than the
406	* PGM one) because of the deadlock risk when we have to delegating the
407	* task to an EMT.
408	* @thread Any.
409	*/
410	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
411	{
412	AssertPtr(ppv);
413	AssertPtr(pLock);
414
415	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
416
417	int rc = pgmLock(pVM);
418	AssertRCReturn(rc, rc);
419
420	/*
421	* Query the Physical TLB entry for the page (may fail).
422	*/
423	PPGMPAGEMAPTLBE pTlbe;
424	rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
425	if (RT_SUCCESS(rc))
426	{
427	PPGMPAGE pPage = pTlbe->pPage;
428	if (PGM_PAGE_IS_MMIO(pPage))
429	rc = VERR_PGM_PHYS_PAGE_RESERVED;
430	else
431	{
432	/*
433	* If the page is shared, the zero page, or being write monitored
434	* it must be converted to an page that's writable if possible.
435	* This has to be done on an EMT.
436	*/
437	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
438	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
439	\|\| pgmPoolIsDirtyPage(pVM, GCPhys)
440	#endif
441	\|\| RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
442	{
443	pgmUnlock(pVM);
444
445	return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
446	pVM, &GCPhys, ppv, pLock);
447	}
448
449	/*
450	* Now, just perform the locking and calculate the return address.
451	*/
452	PPGMPAGEMAP pMap = pTlbe->pMap;
453	if (pMap)
454	pMap->cRefs++;
455
456	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
457	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
458	{
459	if (cLocks == 0)
460	pVM->pgm.s.cWriteLockedPages++;
461	PGM_PAGE_INC_WRITE_LOCKS(pPage);
462	}
463	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
464	{
465	PGM_PAGE_INC_WRITE_LOCKS(pPage);
466	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
467	if (pMap)
468	pMap->cRefs++; /* Extra ref to prevent it from going away. */
469	}
470
471	ppv = (void )((uintptr_t)pTlbe->pv \| (GCPhys & PAGE_OFFSET_MASK));
472	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
473	pLock->pvMap = pMap;
474	}
475	}
476
477	pgmUnlock(pVM);
478	return rc;
479	}
480
481
482	/**
483	* Requests the mapping of a guest page into ring-3, external threads.
484	*
485	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
486	* release it.
487	*
488	* @returns VBox status code.
489	* @retval VINF_SUCCESS on success.
490	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
491	* backing or if the page as an active ALL access handler. The caller
492	* must fall back on using PGMPhysRead.
493	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
494	*
495	* @param pVM The VM handle.
496	* @param GCPhys The guest physical address of the page that should be mapped.
497	* @param ppv Where to store the address corresponding to GCPhys.
498	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
499	*
500	* @remark Avoid calling this API from within critical sections (other than
501	* the PGM one) because of the deadlock risk.
502	* @thread Any.
503	*/
504	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
505	{
506	int rc = pgmLock(pVM);
507	AssertRCReturn(rc, rc);
508
509	/*
510	* Query the Physical TLB entry for the page (may fail).
511	*/
512	PPGMPAGEMAPTLBE pTlbe;
513	rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
514	if (RT_SUCCESS(rc))
515	{
516	PPGMPAGE pPage = pTlbe->pPage;
517	#if 1
518	/* MMIO pages doesn't have any readable backing. */
519	if (PGM_PAGE_IS_MMIO(pPage))
520	rc = VERR_PGM_PHYS_PAGE_RESERVED;
521	#else
522	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
523	rc = VERR_PGM_PHYS_PAGE_RESERVED;
524	#endif
525	else
526	{
527	/*
528	* Now, just perform the locking and calculate the return address.
529	*/
530	PPGMPAGEMAP pMap = pTlbe->pMap;
531	if (pMap)
532	pMap->cRefs++;
533
534	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
535	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
536	{
537	if (cLocks == 0)
538	pVM->pgm.s.cReadLockedPages++;
539	PGM_PAGE_INC_READ_LOCKS(pPage);
540	}
541	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
542	{
543	PGM_PAGE_INC_READ_LOCKS(pPage);
544	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
545	if (pMap)
546	pMap->cRefs++; /* Extra ref to prevent it from going away. */
547	}
548
549	ppv = (void )((uintptr_t)pTlbe->pv \| (GCPhys & PAGE_OFFSET_MASK));
550	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
551	pLock->pvMap = pMap;
552	}
553	}
554
555	pgmUnlock(pVM);
556	return rc;
557	}
558
559
560	/**
561	* Relinks the RAM ranges using the pSelfRC and pSelfR0 pointers.
562	*
563	* Called when anything was relocated.
564	*
565	* @param pVM Pointer to the shared VM structure.
566	*/
567	void pgmR3PhysRelinkRamRanges(PVM pVM)
568	{
569	PPGMRAMRANGE pCur;
570
571	#ifdef VBOX_STRICT
572	for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
573	{
574	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfR0 == MMHyperCCToR0(pVM, pCur));
575	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfRC == MMHyperCCToRC(pVM, pCur));
576	Assert((pCur->GCPhys & PAGE_OFFSET_MASK) == 0);
577	Assert((pCur->GCPhysLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
578	Assert((pCur->cb & PAGE_OFFSET_MASK) == 0);
579	Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
580	for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesR3; pCur2; pCur2 = pCur2->pNextR3)
581	Assert( pCur2 == pCur
582	\|\| strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
583	}
584	#endif
585
586	pCur = pVM->pgm.s.pRamRangesR3;
587	if (pCur)
588	{
589	pVM->pgm.s.pRamRangesR0 = pCur->pSelfR0;
590	pVM->pgm.s.pRamRangesRC = pCur->pSelfRC;
591
592	for (; pCur->pNextR3; pCur = pCur->pNextR3)
593	{
594	pCur->pNextR0 = pCur->pNextR3->pSelfR0;
595	pCur->pNextRC = pCur->pNextR3->pSelfRC;
596	}
597
598	Assert(pCur->pNextR0 == NIL_RTR0PTR);
599	Assert(pCur->pNextRC == NIL_RTRCPTR);
600	}
601	else
602	{
603	Assert(pVM->pgm.s.pRamRangesR0 == NIL_RTR0PTR);
604	Assert(pVM->pgm.s.pRamRangesRC == NIL_RTRCPTR);
605	}
606	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
607	}
608
609
610	/**
611	* Links a new RAM range into the list.
612	*
613	* @param pVM Pointer to the shared VM structure.
614	* @param pNew Pointer to the new list entry.
615	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
616	*/
617	static void pgmR3PhysLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, PPGMRAMRANGE pPrev)
618	{
619	AssertMsg(pNew->pszDesc, ("%RGp-%RGp\n", pNew->GCPhys, pNew->GCPhysLast));
620	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfR0 == MMHyperCCToR0(pVM, pNew));
621	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfRC == MMHyperCCToRC(pVM, pNew));
622
623	pgmLock(pVM);
624
625	PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesR3;
626	pNew->pNextR3 = pRam;
627	pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
628	pNew->pNextRC = pRam ? pRam->pSelfRC : NIL_RTRCPTR;
629
630	if (pPrev)
631	{
632	pPrev->pNextR3 = pNew;
633	pPrev->pNextR0 = pNew->pSelfR0;
634	pPrev->pNextRC = pNew->pSelfRC;
635	}
636	else
637	{
638	pVM->pgm.s.pRamRangesR3 = pNew;
639	pVM->pgm.s.pRamRangesR0 = pNew->pSelfR0;
640	pVM->pgm.s.pRamRangesRC = pNew->pSelfRC;
641	}
642	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
643
644	pgmUnlock(pVM);
645	}
646
647
648	/**
649	* Unlink an existing RAM range from the list.
650	*
651	* @param pVM Pointer to the shared VM structure.
652	* @param pRam Pointer to the new list entry.
653	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
654	*/
655	static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
656	{
657	Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesR3 == pRam);
658	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfR0 == MMHyperCCToR0(pVM, pRam));
659	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfRC == MMHyperCCToRC(pVM, pRam));
660
661	pgmLock(pVM);
662
663	PPGMRAMRANGE pNext = pRam->pNextR3;
664	if (pPrev)
665	{
666	pPrev->pNextR3 = pNext;
667	pPrev->pNextR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
668	pPrev->pNextRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
669	}
670	else
671	{
672	Assert(pVM->pgm.s.pRamRangesR3 == pRam);
673	pVM->pgm.s.pRamRangesR3 = pNext;
674	pVM->pgm.s.pRamRangesR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
675	pVM->pgm.s.pRamRangesRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
676	}
677	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
678
679	pgmUnlock(pVM);
680	}
681
682
683	/**
684	* Unlink an existing RAM range from the list.
685	*
686	* @param pVM Pointer to the shared VM structure.
687	* @param pRam Pointer to the new list entry.
688	*/
689	static void pgmR3PhysUnlinkRamRange(PVM pVM, PPGMRAMRANGE pRam)
690	{
691	pgmLock(pVM);
692
693	/* find prev. */
694	PPGMRAMRANGE pPrev = NULL;
695	PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesR3;
696	while (pCur != pRam)
697	{
698	pPrev = pCur;
699	pCur = pCur->pNextR3;
700	}
701	AssertFatal(pCur);
702
703	pgmR3PhysUnlinkRamRange2(pVM, pRam, pPrev);
704
705	pgmUnlock(pVM);
706	}
707
708
709	/**
710	* Frees a range of pages, replacing them with ZERO pages of the specified type.
711	*
712	* @returns VBox status code.
713	* @param pVM The VM handle.
714	* @param pRam The RAM range in which the pages resides.
715	* @param GCPhys The address of the first page.
716	* @param GCPhysLast The address of the last page.
717	* @param uType The page type to replace then with.
718	*/
719	static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, uint8_t uType)
720	{
721	uint32_t cPendingPages = 0;
722	PGMMFREEPAGESREQ pReq;
723	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
724	AssertLogRelRCReturn(rc, rc);
725
726	/* Itegerate the pages. */
727	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
728	uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> PAGE_SHIFT) + 1;
729	while (cPagesLeft-- > 0)
730	{
731	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
732	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
733
734	PGM_PAGE_SET_TYPE(pPageDst, uType);
735
736	GCPhys += PAGE_SIZE;
737	pPageDst++;
738	}
739
740	if (cPendingPages)
741	{
742	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
743	AssertLogRelRCReturn(rc, rc);
744	}
745	GMMR3FreePagesCleanup(pReq);
746
747	return rc;
748	}
749
750
751	/**
752	* PGMR3PhysRegisterRam worker that initializes and links a RAM range.
753	*
754	* @param pVM The VM handle.
755	* @param pNew The new RAM range.
756	* @param GCPhys The address of the RAM range.
757	* @param GCPhysLast The last address of the RAM range.
758	* @param RCPtrNew The RC address if the range is floating. NIL_RTRCPTR
759	* if in HMA.
760	* @param R0PtrNew Ditto for R0.
761	* @param pszDesc The description.
762	* @param pPrev The previous RAM range (for linking).
763	*/
764	static void pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
765	RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
766	{
767	/*
768	* Initialize the range.
769	*/
770	pNew->pSelfR0 = R0PtrNew != NIL_RTR0PTR ? R0PtrNew : MMHyperCCToR0(pVM, pNew);
771	pNew->pSelfRC = RCPtrNew != NIL_RTRCPTR ? RCPtrNew : MMHyperCCToRC(pVM, pNew);
772	pNew->GCPhys = GCPhys;
773	pNew->GCPhysLast = GCPhysLast;
774	pNew->cb = GCPhysLast - GCPhys + 1;
775	pNew->pszDesc = pszDesc;
776	pNew->fFlags = RCPtrNew != NIL_RTRCPTR ? PGM_RAM_RANGE_FLAGS_FLOATING : 0;
777	pNew->pvR3 = NULL;
778	pNew->paLSPages = NULL;
779
780	uint32_t const cPages = pNew->cb >> PAGE_SHIFT;
781	RTGCPHYS iPage = cPages;
782	while (iPage-- > 0)
783	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
784
785	/* Update the page count stats. */
786	pVM->pgm.s.cZeroPages += cPages;
787	pVM->pgm.s.cAllPages += cPages;
788
789	/*
790	* Link it.
791	*/
792	pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
793	}
794
795
796	/**
797	* Relocate a floating RAM range.
798	*
799	* @copydoc FNPGMRELOCATE.
800	*/
801	static DECLCALLBACK(bool) pgmR3PhysRamRangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser)
802	{
803	PPGMRAMRANGE pRam = (PPGMRAMRANGE)pvUser;
804	Assert(pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
805	Assert(pRam->pSelfRC == GCPtrOld + PAGE_SIZE);
806
807	switch (enmMode)
808	{
809	case PGMRELOCATECALL_SUGGEST:
810	return true;
811	case PGMRELOCATECALL_RELOCATE:
812	{
813	/* Update myself and then relink all the ranges. */
814	pgmLock(pVM);
815	pRam->pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE);
816	pgmR3PhysRelinkRamRanges(pVM);
817	pgmUnlock(pVM);
818	return true;
819	}
820
821	default:
822	AssertFailedReturn(false);
823	}
824	}
825
826
827	/**
828	* PGMR3PhysRegisterRam worker that registers a high chunk.
829	*
830	* @returns VBox status code.
831	* @param pVM The VM handle.
832	* @param GCPhys The address of the RAM.
833	* @param cRamPages The number of RAM pages to register.
834	* @param cbChunk The size of the PGMRAMRANGE guest mapping.
835	* @param iChunk The chunk number.
836	* @param pszDesc The RAM range description.
837	* @param ppPrev Previous RAM range pointer. In/Out.
838	*/
839	static int pgmR3PhysRegisterHighRamChunk(PVM pVM, RTGCPHYS GCPhys, uint32_t cRamPages,
840	uint32_t cbChunk, uint32_t iChunk, const char *pszDesc,
841	PPGMRAMRANGE *ppPrev)
842	{
843	const char *pszDescChunk = iChunk == 0
844	? pszDesc
845	: MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, iChunk + 1);
846	AssertReturn(pszDescChunk, VERR_NO_MEMORY);
847
848	/*
849	* Allocate memory for the new chunk.
850	*/
851	size_t const cChunkPages = RT_ALIGN_Z(RT_UOFFSETOF(PGMRAMRANGE, aPages[cRamPages]), PAGE_SIZE) >> PAGE_SHIFT;
852	PSUPPAGE paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
853	AssertReturn(paChunkPages, VERR_NO_TMP_MEMORY);
854	RTR0PTR R0PtrChunk = NIL_RTR0PTR;
855	void *pvChunk = NULL;
856	int rc = SUPR3PageAllocEx(cChunkPages, 0 /fFlags/, &pvChunk,
857	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
858	VMMIsHwVirtExtForced(pVM) ? &R0PtrChunk : NULL,
859	#else
860	NULL,
861	#endif
862	paChunkPages);
863	if (RT_SUCCESS(rc))
864	{
865	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
866	if (!VMMIsHwVirtExtForced(pVM))
867	R0PtrChunk = NIL_RTR0PTR;
868	#else
869	R0PtrChunk = (uintptr_t)pvChunk;
870	#endif
871	memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
872
873	PPGMRAMRANGE pNew = (PPGMRAMRANGE)pvChunk;
874
875	/*
876	* Create a mapping and map the pages into it.
877	* We push these in below the HMA.
878	*/
879	RTGCPTR GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
880	rc = PGMR3MapPT(pVM, GCPtrChunkMap, cbChunk, 0 /fFlags/, pgmR3PhysRamRangeRelocate, pNew, pszDescChunk);
881	if (RT_SUCCESS(rc))
882	{
883	pVM->pgm.s.GCPtrPrevRamRangeMapping = GCPtrChunkMap;
884
885	RTGCPTR const GCPtrChunk = GCPtrChunkMap + PAGE_SIZE;
886	RTGCPTR GCPtrPage = GCPtrChunk;
887	for (uint32_t iPage = 0; iPage < cChunkPages && RT_SUCCESS(rc); iPage++, GCPtrPage += PAGE_SIZE)
888	rc = PGMMap(pVM, GCPtrPage, paChunkPages[iPage].Phys, PAGE_SIZE, 0);
889	if (RT_SUCCESS(rc))
890	{
891	/*
892	* Ok, init and link the range.
893	*/
894	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
895	(RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
896	*ppPrev = pNew;
897	}
898	}
899
900	if (RT_FAILURE(rc))
901	SUPR3PageFreeEx(pvChunk, cChunkPages);
902	}
903
904	RTMemTmpFree(paChunkPages);
905	return rc;
906	}
907
908
909	/**
910	* Sets up a range RAM.
911	*
912	* This will check for conflicting registrations, make a resource
913	* reservation for the memory (with GMM), and setup the per-page
914	* tracking structures (PGMPAGE).
915	*
916	* @returns VBox stutus code.
917	* @param pVM Pointer to the shared VM structure.
918	* @param GCPhys The physical address of the RAM.
919	* @param cb The size of the RAM.
920	* @param pszDesc The description - not copied, so, don't free or change it.
921	*/
922	VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
923	{
924	/*
925	* Validate input.
926	*/
927	Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
928	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
929	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
930	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
931	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
932	AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
933	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
934	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
935
936	pgmLock(pVM);
937
938	/*
939	* Find range location and check for conflicts.
940	* (We don't lock here because the locking by EMT is only required on update.)
941	*/
942	PPGMRAMRANGE pPrev = NULL;
943	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
944	while (pRam && GCPhysLast >= pRam->GCPhys)
945	{
946	if ( GCPhysLast >= pRam->GCPhys
947	&& GCPhys <= pRam->GCPhysLast)
948	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
949	GCPhys, GCPhysLast, pszDesc,
950	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
951	VERR_PGM_RAM_CONFLICT);
952
953	/* next */
954	pPrev = pRam;
955	pRam = pRam->pNextR3;
956	}
957
958	/*
959	* Register it with GMM (the API bitches).
960	*/
961	const RTGCPHYS cPages = cb >> PAGE_SHIFT;
962	int rc = MMR3IncreaseBaseReservation(pVM, cPages);
963	if (RT_FAILURE(rc))
964	{
965	pgmUnlock(pVM);
966	return rc;
967	}
968
969	if ( GCPhys >= _4G
970	&& cPages > 256)
971	{
972	/*
973	* The PGMRAMRANGE structures for the high memory can get very big.
974	* In order to avoid SUPR3PageAllocEx allocation failures due to the
975	* allocation size limit there and also to avoid being unable to find
976	* guest mapping space for them, we split this memory up into 4MB in
977	* (potential) raw-mode configs and 16MB chunks in forced AMD-V/VT-x
978	* mode.
979	*
980	* The first and last page of each mapping are guard pages and marked
981	* not-present. So, we've got 4186112 and 16769024 bytes available for
982	* the PGMRAMRANGE structure.
983	*
984	* Note! The sizes used here will influence the saved state.
985	*/
986	uint32_t cbChunk;
987	uint32_t cPagesPerChunk;
988	if (VMMIsHwVirtExtForced(pVM))
989	{
990	cbChunk = 16U*_1M;
991	cPagesPerChunk = 1048048; /* max ~1048059 */
992	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 1048048 < 16U_1M - PAGE_SIZE 2);
993	}
994	else
995	{
996	cbChunk = 4U*_1M;
997	cPagesPerChunk = 261616; /* max ~261627 */
998	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 261616 < 4U_1M - PAGE_SIZE 2);
999	}
1000	AssertRelease(RT_UOFFSETOF(PGMRAMRANGE, aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
1001
1002	RTGCPHYS cPagesLeft = cPages;
1003	RTGCPHYS GCPhysChunk = GCPhys;
1004	uint32_t iChunk = 0;
1005	while (cPagesLeft > 0)
1006	{
1007	uint32_t cPagesInChunk = cPagesLeft;
1008	if (cPagesInChunk > cPagesPerChunk)
1009	cPagesInChunk = cPagesPerChunk;
1010
1011	rc = pgmR3PhysRegisterHighRamChunk(pVM, GCPhysChunk, cPagesInChunk, cbChunk, iChunk, pszDesc, &pPrev);
1012	AssertRCReturn(rc, rc);
1013
1014	/* advance */
1015	GCPhysChunk += (RTGCPHYS)cPagesInChunk << PAGE_SHIFT;
1016	cPagesLeft -= cPagesInChunk;
1017	iChunk++;
1018	}
1019	}
1020	else
1021	{
1022	/*
1023	* Allocate, initialize and link the new RAM range.
1024	*/
1025	const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
1026	PPGMRAMRANGE pNew;
1027	rc = MMR3HyperAllocOnceNoRel(pVM, cbRamRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
1028	AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
1029
1030	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, NIL_RTR0PTR, pszDesc, pPrev);
1031	}
1032	pgmUnlock(pVM);
1033
1034	/*
1035	* Notify REM.
1036	*/
1037	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_RAM);
1038
1039	return VINF_SUCCESS;
1040	}
1041
1042
1043	/**
1044	* Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
1045	*
1046	* We do this late in the init process so that all the ROM and MMIO ranges have
1047	* been registered already and we don't go wasting memory on them.
1048	*
1049	* @returns VBox status code.
1050	*
1051	* @param pVM Pointer to the shared VM structure.
1052	*/
1053	int pgmR3PhysRamPreAllocate(PVM pVM)
1054	{
1055	Assert(pVM->pgm.s.fRamPreAlloc);
1056	Log(("pgmR3PhysRamPreAllocate: enter\n"));
1057
1058	/*
1059	* Walk the RAM ranges and allocate all RAM pages, halt at
1060	* the first allocation error.
1061	*/
1062	uint64_t cPages = 0;
1063	uint64_t NanoTS = RTTimeNanoTS();
1064	pgmLock(pVM);
1065	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
1066	{
1067	PPGMPAGE pPage = &pRam->aPages[0];
1068	RTGCPHYS GCPhys = pRam->GCPhys;
1069	uint32_t cLeft = pRam->cb >> PAGE_SHIFT;
1070	while (cLeft-- > 0)
1071	{
1072	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1073	{
1074	switch (PGM_PAGE_GET_STATE(pPage))
1075	{
1076	case PGM_PAGE_STATE_ZERO:
1077	{
1078	int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
1079	if (RT_FAILURE(rc))
1080	{
1081	LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
1082	pgmUnlock(pVM);
1083	return rc;
1084	}
1085	cPages++;
1086	break;
1087	}
1088
1089	case PGM_PAGE_STATE_ALLOCATED:
1090	case PGM_PAGE_STATE_WRITE_MONITORED:
1091	case PGM_PAGE_STATE_SHARED:
1092	/* nothing to do here. */
1093	break;
1094	}
1095	}
1096
1097	/* next */
1098	pPage++;
1099	GCPhys += PAGE_SIZE;
1100	}
1101	}
1102	pgmUnlock(pVM);
1103	NanoTS = RTTimeNanoTS() - NanoTS;
1104
1105	LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
1106	Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
1107	return VINF_SUCCESS;
1108	}
1109
1110
1111	/**
1112	* Resets (zeros) the RAM.
1113	*
1114	* ASSUMES that the caller owns the PGM lock.
1115	*
1116	* @returns VBox status code.
1117	* @param pVM Pointer to the shared VM structure.
1118	*/
1119	int pgmR3PhysRamReset(PVM pVM)
1120	{
1121	Assert(PGMIsLockOwner(pVM));
1122	/*
1123	* We batch up pages before freeing them.
1124	*/
1125	uint32_t cPendingPages = 0;
1126	PGMMFREEPAGESREQ pReq;
1127	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1128	AssertLogRelRCReturn(rc, rc);
1129
1130	/*
1131	* Walk the ram ranges.
1132	*/
1133	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
1134	{
1135	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1136	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1137
1138	if (!pVM->pgm.s.fRamPreAlloc)
1139	{
1140	/* Replace all RAM pages by ZERO pages. */
1141	while (iPage-- > 0)
1142	{
1143	PPGMPAGE pPage = &pRam->aPages[iPage];
1144	switch (PGM_PAGE_GET_TYPE(pPage))
1145	{
1146	case PGMPAGETYPE_RAM:
1147	if (!PGM_PAGE_IS_ZERO(pPage))
1148	{
1149	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1150	AssertLogRelRCReturn(rc, rc);
1151	}
1152	break;
1153
1154	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1155	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1156	break;
1157
1158	case PGMPAGETYPE_MMIO2:
1159	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
1160	case PGMPAGETYPE_ROM:
1161	case PGMPAGETYPE_MMIO:
1162	break;
1163	default:
1164	AssertFailed();
1165	}
1166	} /* for each page */
1167	}
1168	else
1169	{
1170	/* Zero the memory. */
1171	while (iPage-- > 0)
1172	{
1173	PPGMPAGE pPage = &pRam->aPages[iPage];
1174	switch (PGM_PAGE_GET_TYPE(pPage))
1175	{
1176	case PGMPAGETYPE_RAM:
1177	switch (PGM_PAGE_GET_STATE(pPage))
1178	{
1179	case PGM_PAGE_STATE_ZERO:
1180	break;
1181	case PGM_PAGE_STATE_SHARED:
1182	case PGM_PAGE_STATE_WRITE_MONITORED:
1183	rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1184	AssertLogRelRCReturn(rc, rc);
1185	case PGM_PAGE_STATE_ALLOCATED:
1186	{
1187	void *pvPage;
1188	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1189	AssertLogRelRCReturn(rc, rc);
1190	ASMMemZeroPage(pvPage);
1191	break;
1192	}
1193	}
1194	break;
1195
1196	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1197	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1198	break;
1199
1200	case PGMPAGETYPE_MMIO2:
1201	case PGMPAGETYPE_ROM_SHADOW:
1202	case PGMPAGETYPE_ROM:
1203	case PGMPAGETYPE_MMIO:
1204	break;
1205	default:
1206	AssertFailed();
1207
1208	}
1209	} /* for each page */
1210	}
1211
1212	}
1213
1214	/*
1215	* Finish off any pages pending freeing.
1216	*/
1217	if (cPendingPages)
1218	{
1219	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
1220	AssertLogRelRCReturn(rc, rc);
1221	}
1222	GMMR3FreePagesCleanup(pReq);
1223
1224	return VINF_SUCCESS;
1225	}
1226
1227
1228	/**
1229	* This is the interface IOM is using to register an MMIO region.
1230	*
1231	* It will check for conflicts and ensure that a RAM range structure
1232	* is present before calling the PGMR3HandlerPhysicalRegister API to
1233	* register the callbacks.
1234	*
1235	* @returns VBox status code.
1236	*
1237	* @param pVM Pointer to the shared VM structure.
1238	* @param GCPhys The start of the MMIO region.
1239	* @param cb The size of the MMIO region.
1240	* @param pfnHandlerR3 The address of the ring-3 handler. (IOMR3MMIOHandler)
1241	* @param pvUserR3 The user argument for R3.
1242	* @param pfnHandlerR0 The address of the ring-0 handler. (IOMMMIOHandler)
1243	* @param pvUserR0 The user argument for R0.
1244	* @param pfnHandlerRC The address of the RC handler. (IOMMMIOHandler)
1245	* @param pvUserRC The user argument for RC.
1246	* @param pszDesc The description of the MMIO region.
1247	*/
1248	VMMR3DECL(int) PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb,
1249	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3, RTR3PTR pvUserR3,
1250	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0, RTR0PTR pvUserR0,
1251	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC, RTRCPTR pvUserRC,
1252	R3PTRTYPE(const char *) pszDesc)
1253	{
1254	/*
1255	* Assert on some assumption.
1256	*/
1257	VM_ASSERT_EMT(pVM);
1258	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
1259	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
1260	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1261	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
1262
1263	/*
1264	* Make sure there's a RAM range structure for the region.
1265	*/
1266	int rc;
1267	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1268	bool fRamExists = false;
1269	PPGMRAMRANGE pRamPrev = NULL;
1270	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
1271	while (pRam && GCPhysLast >= pRam->GCPhys)
1272	{
1273	if ( GCPhysLast >= pRam->GCPhys
1274	&& GCPhys <= pRam->GCPhysLast)
1275	{
1276	/* Simplification: all within the same range. */
1277	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
1278	&& GCPhysLast <= pRam->GCPhysLast,
1279	("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
1280	GCPhys, GCPhysLast, pszDesc,
1281	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
1282	VERR_PGM_RAM_CONFLICT);
1283
1284	/* Check that it's all RAM or MMIO pages. */
1285	PCPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
1286	uint32_t cLeft = cb >> PAGE_SHIFT;
1287	while (cLeft-- > 0)
1288	{
1289	AssertLogRelMsgReturn( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1290	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO,
1291	("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
1292	GCPhys, GCPhysLast, pszDesc, PGM_PAGE_GET_TYPE(pPage), pRam->pszDesc),
1293	VERR_PGM_RAM_CONFLICT);
1294	pPage++;
1295	}
1296
1297	/* Looks good. */
1298	fRamExists = true;
1299	break;
1300	}
1301
1302	/* next */
1303	pRamPrev = pRam;
1304	pRam = pRam->pNextR3;
1305	}
1306	PPGMRAMRANGE pNew;
1307	if (fRamExists)
1308	{
1309	pNew = NULL;
1310
1311	/*
1312	* Make all the pages in the range MMIO/ZERO pages, freeing any
1313	* RAM pages currently mapped here. This might not be 100% correct
1314	* for PCI memory, but we're doing the same thing for MMIO2 pages.
1315	*/
1316	rc = pgmLock(pVM);
1317	if (RT_SUCCESS(rc))
1318	{
1319	rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
1320	pgmUnlock(pVM);
1321	}
1322	AssertRCReturn(rc, rc);
1323	}
1324	else
1325	{
1326	pgmLock(pVM);
1327
1328	/*
1329	* No RAM range, insert an ad hoc one.
1330	*
1331	* Note that we don't have to tell REM about this range because
1332	* PGMHandlerPhysicalRegisterEx will do that for us.
1333	*/
1334	Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
1335
1336	const uint32_t cPages = cb >> PAGE_SHIFT;
1337	const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
1338	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), 16, MM_TAG_PGM_PHYS, (void **)&pNew);
1339	AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
1340
1341	/* Initialize the range. */
1342	pNew->pSelfR0 = MMHyperCCToR0(pVM, pNew);
1343	pNew->pSelfRC = MMHyperCCToRC(pVM, pNew);
1344	pNew->GCPhys = GCPhys;
1345	pNew->GCPhysLast = GCPhysLast;
1346	pNew->cb = cb;
1347	pNew->pszDesc = pszDesc;
1348	pNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO;
1349	pNew->pvR3 = NULL;
1350	pNew->paLSPages = NULL;
1351
1352	uint32_t iPage = cPages;
1353	while (iPage-- > 0)
1354	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
1355	Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
1356
1357	/* update the page count stats. */
1358	pVM->pgm.s.cZeroPages += cPages;
1359	pVM->pgm.s.cAllPages += cPages;
1360
1361	/* link it */
1362	pgmR3PhysLinkRamRange(pVM, pNew, pRamPrev);
1363
1364	pgmUnlock(pVM);
1365	}
1366
1367	/*
1368	* Register the access handler.
1369	*/
1370	rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_MMIO, GCPhys, GCPhysLast,
1371	pfnHandlerR3, pvUserR3,
1372	pfnHandlerR0, pvUserR0,
1373	pfnHandlerRC, pvUserRC, pszDesc);
1374	if ( RT_FAILURE(rc)
1375	&& !fRamExists)
1376	{
1377	pVM->pgm.s.cZeroPages -= cb >> PAGE_SHIFT;
1378	pVM->pgm.s.cAllPages -= cb >> PAGE_SHIFT;
1379
1380	/* remove the ad hoc range. */
1381	pgmR3PhysUnlinkRamRange2(pVM, pNew, pRamPrev);
1382	pNew->cb = pNew->GCPhys = pNew->GCPhysLast = NIL_RTGCPHYS;
1383	MMHyperFree(pVM, pRam);
1384	}
1385
1386	return rc;
1387	}
1388
1389
1390	/**
1391	* This is the interface IOM is using to register an MMIO region.
1392	*
1393	* It will take care of calling PGMHandlerPhysicalDeregister and clean up
1394	* any ad hoc PGMRAMRANGE left behind.
1395	*
1396	* @returns VBox status code.
1397	* @param pVM Pointer to the shared VM structure.
1398	* @param GCPhys The start of the MMIO region.
1399	* @param cb The size of the MMIO region.
1400	*/
1401	VMMR3DECL(int) PGMR3PhysMMIODeregister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
1402	{
1403	VM_ASSERT_EMT(pVM);
1404
1405	/*
1406	* First deregister the handler, then check if we should remove the ram range.
1407	*/
1408	int rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
1409	if (RT_SUCCESS(rc))
1410	{
1411	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1412	PPGMRAMRANGE pRamPrev = NULL;
1413	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
1414	while (pRam && GCPhysLast >= pRam->GCPhys)
1415	{
1416	/** @todo We're being a bit too careful here. rewrite. */
1417	if ( GCPhysLast == pRam->GCPhysLast
1418	&& GCPhys == pRam->GCPhys)
1419	{
1420	Assert(pRam->cb == cb);
1421
1422	/*
1423	* See if all the pages are dead MMIO pages.
1424	*/
1425	uint32_t const cPages = cb >> PAGE_SHIFT;
1426	bool fAllMMIO = true;
1427	uint32_t iPage = 0;
1428	uint32_t cLeft = cPages;
1429	while (cLeft-- > 0)
1430	{
1431	PPGMPAGE pPage = &pRam->aPages[iPage];
1432	if ( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO
1433	/\|\| not-out-of-action later /)
1434	{
1435	fAllMMIO = false;
1436	Assert(PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO2_ALIAS_MMIO);
1437	AssertMsgFailed(("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
1438	break;
1439	}
1440	Assert(PGM_PAGE_IS_ZERO(pPage));
1441	pPage++;
1442	}
1443	if (fAllMMIO)
1444	{
1445	/*
1446	* Ad-hoc range, unlink and free it.
1447	*/
1448	Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
1449	GCPhys, GCPhysLast, pRam->pszDesc));
1450
1451	pVM->pgm.s.cAllPages -= cPages;
1452	pVM->pgm.s.cZeroPages -= cPages;
1453
1454	pgmR3PhysUnlinkRamRange2(pVM, pRam, pRamPrev);
1455	pRam->cb = pRam->GCPhys = pRam->GCPhysLast = NIL_RTGCPHYS;
1456	MMHyperFree(pVM, pRam);
1457	break;
1458	}
1459	}
1460
1461	/*
1462	* Range match? It will all be within one range (see PGMAllHandler.cpp).
1463	*/
1464	if ( GCPhysLast >= pRam->GCPhys
1465	&& GCPhys <= pRam->GCPhysLast)
1466	{
1467	Assert(GCPhys >= pRam->GCPhys);
1468	Assert(GCPhysLast <= pRam->GCPhysLast);
1469
1470	/*
1471	* Turn the pages back into RAM pages.
1472	*/
1473	uint32_t iPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
1474	uint32_t cLeft = cb >> PAGE_SHIFT;
1475	while (cLeft--)
1476	{
1477	PPGMPAGE pPage = &pRam->aPages[iPage];
1478	AssertMsg(PGM_PAGE_IS_MMIO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
1479	AssertMsg(PGM_PAGE_IS_ZERO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
1480	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO)
1481	PGM_PAGE_SET_TYPE(pPage, PGMPAGETYPE_RAM);
1482	}
1483	break;
1484	}
1485
1486	/* next */
1487	pRamPrev = pRam;
1488	pRam = pRam->pNextR3;
1489	}
1490	}
1491
1492	return rc;
1493	}
1494
1495
1496	/**
1497	* Locate a MMIO2 range.
1498	*
1499	* @returns Pointer to the MMIO2 range.
1500	* @param pVM Pointer to the shared VM structure.
1501	* @param pDevIns The device instance owning the region.
1502	* @param iRegion The region.
1503	*/
1504	DECLINLINE(PPGMMMIO2RANGE) pgmR3PhysMMIO2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
1505	{
1506	/*
1507	* Search the list.
1508	*/
1509	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
1510	if ( pCur->pDevInsR3 == pDevIns
1511	&& pCur->iRegion == iRegion)
1512	return pCur;
1513	return NULL;
1514	}
1515
1516
1517	/**
1518	* Allocate and register an MMIO2 region.
1519	*
1520	* As mentioned elsewhere, MMIO2 is just RAM spelled differently. It's
1521	* RAM associated with a device. It is also non-shared memory with a
1522	* permanent ring-3 mapping and page backing (presently).
1523	*
1524	* A MMIO2 range may overlap with base memory if a lot of RAM
1525	* is configured for the VM, in which case we'll drop the base
1526	* memory pages. Presently we will make no attempt to preserve
1527	* anything that happens to be present in the base memory that
1528	* is replaced, this is of course incorrectly but it's too much
1529	* effort.
1530	*
1531	* @returns VBox status code.
1532	* @retval VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the memory.
1533	* @retval VERR_ALREADY_EXISTS if the region already exists.
1534	*
1535	* @param pVM Pointer to the shared VM structure.
1536	* @param pDevIns The device instance owning the region.
1537	* @param iRegion The region number. If the MMIO2 memory is a PCI I/O region
1538	* this number has to be the number of that region. Otherwise
1539	* it can be any number safe UINT8_MAX.
1540	* @param cb The size of the region. Must be page aligned.
1541	* @param fFlags Reserved for future use, must be zero.
1542	* @param ppv Where to store the pointer to the ring-3 mapping of the memory.
1543	* @param pszDesc The description.
1544	*/
1545	VMMR3DECL(int) PGMR3PhysMMIO2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS cb, uint32_t fFlags, void *ppv, const char pszDesc)
1546	{
1547	/*
1548	* Validate input.
1549	*/
1550	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1551	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
1552	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
1553	AssertPtrReturn(ppv, VERR_INVALID_POINTER);
1554	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1555	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
1556	AssertReturn(pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion) == NULL, VERR_ALREADY_EXISTS);
1557	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
1558	AssertReturn(cb, VERR_INVALID_PARAMETER);
1559	AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
1560
1561	const uint32_t cPages = cb >> PAGE_SHIFT;
1562	AssertLogRelReturn(((RTGCPHYS)cPages << PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
1563	AssertLogRelReturn(cPages <= INT32_MAX / 2, VERR_NO_MEMORY);
1564
1565	/*
1566	* For the 2nd+ instance, mangle the description string so it's unique.
1567	*/
1568	if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
1569	{
1570	pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
1571	if (!pszDesc)
1572	return VERR_NO_MEMORY;
1573	}
1574
1575	/*
1576	* Try reserve and allocate the backing memory first as this is what is
1577	* most likely to fail.
1578	*/
1579	int rc = MMR3AdjustFixedReservation(pVM, cPages, pszDesc);
1580	if (RT_SUCCESS(rc))
1581	{
1582	void *pvPages;
1583	PSUPPAGE paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(SUPPAGE));
1584	if (RT_SUCCESS(rc))
1585	rc = SUPR3PageAllocEx(cPages, 0 /fFlags/, &pvPages, NULL /pR0Ptr/, paPages);
1586	if (RT_SUCCESS(rc))
1587	{
1588	memset(pvPages, 0, cPages * PAGE_SIZE);
1589
1590	/*
1591	* Create the MMIO2 range record for it.
1592	*/
1593	const size_t cbRange = RT_OFFSETOF(PGMMMIO2RANGE, RamRange.aPages[cPages]);
1594	PPGMMMIO2RANGE pNew;
1595	rc = MMR3HyperAllocOnceNoRel(pVM, cbRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
1596	AssertLogRelMsgRC(rc, ("cbRamRange=%zu\n", cbRange));
1597	if (RT_SUCCESS(rc))
1598	{
1599	pNew->pDevInsR3 = pDevIns;
1600	pNew->pvR3 = pvPages;
1601	//pNew->pNext = NULL;
1602	//pNew->fMapped = false;
1603	//pNew->fOverlapping = false;
1604	pNew->iRegion = iRegion;
1605	pNew->idSavedState = UINT8_MAX;
1606	pNew->RamRange.pSelfR0 = MMHyperCCToR0(pVM, &pNew->RamRange);
1607	pNew->RamRange.pSelfRC = MMHyperCCToRC(pVM, &pNew->RamRange);
1608	pNew->RamRange.GCPhys = NIL_RTGCPHYS;
1609	pNew->RamRange.GCPhysLast = NIL_RTGCPHYS;
1610	pNew->RamRange.pszDesc = pszDesc;
1611	pNew->RamRange.cb = cb;
1612	pNew->RamRange.fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2;
1613	pNew->RamRange.pvR3 = pvPages;
1614	//pNew->RamRange.paLSPages = NULL;
1615
1616	uint32_t iPage = cPages;
1617	while (iPage-- > 0)
1618	{
1619	PGM_PAGE_INIT(&pNew->RamRange.aPages[iPage],
1620	paPages[iPage].Phys, NIL_GMM_PAGEID,
1621	PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
1622	}
1623
1624	/* update page count stats */
1625	pVM->pgm.s.cAllPages += cPages;
1626	pVM->pgm.s.cPrivatePages += cPages;
1627
1628	/*
1629	* Link it into the list.
1630	* Since there is no particular order, just push it.
1631	*/
1632	pgmLock(pVM);
1633	pNew->pNextR3 = pVM->pgm.s.pMmio2RangesR3;
1634	pVM->pgm.s.pMmio2RangesR3 = pNew;
1635	pgmUnlock(pVM);
1636
1637	*ppv = pvPages;
1638	RTMemTmpFree(paPages);
1639	return VINF_SUCCESS;
1640	}
1641
1642	SUPR3PageFreeEx(pvPages, cPages);
1643	}
1644	RTMemTmpFree(paPages);
1645	MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pszDesc);
1646	}
1647	if (pDevIns->iInstance > 0)
1648	MMR3HeapFree((void *)pszDesc);
1649	return rc;
1650	}
1651
1652
1653	/**
1654	* Deregisters and frees an MMIO2 region.
1655	*
1656	* Any physical (and virtual) access handlers registered for the region must
1657	* be deregistered before calling this function.
1658	*
1659	* @returns VBox status code.
1660	* @param pVM Pointer to the shared VM structure.
1661	* @param pDevIns The device instance owning the region.
1662	* @param iRegion The region. If it's UINT32_MAX it'll be a wildcard match.
1663	*/
1664	VMMR3DECL(int) PGMR3PhysMMIO2Deregister(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
1665	{
1666	/*
1667	* Validate input.
1668	*/
1669	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1670	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
1671	AssertReturn(iRegion <= UINT8_MAX \|\| iRegion == UINT32_MAX, VERR_INVALID_PARAMETER);
1672
1673	pgmLock(pVM);
1674	int rc = VINF_SUCCESS;
1675	unsigned cFound = 0;
1676	PPGMMMIO2RANGE pPrev = NULL;
1677	PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3;
1678	while (pCur)
1679	{
1680	if ( pCur->pDevInsR3 == pDevIns
1681	&& ( iRegion == UINT32_MAX
1682	\|\| pCur->iRegion == iRegion))
1683	{
1684	cFound++;
1685
1686	/*
1687	* Unmap it if it's mapped.
1688	*/
1689	if (pCur->fMapped)
1690	{
1691	int rc2 = PGMR3PhysMMIO2Unmap(pVM, pCur->pDevInsR3, pCur->iRegion, pCur->RamRange.GCPhys);
1692	AssertRC(rc2);
1693	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
1694	rc = rc2;
1695	}
1696
1697	/*
1698	* Unlink it
1699	*/
1700	PPGMMMIO2RANGE pNext = pCur->pNextR3;
1701	if (pPrev)
1702	pPrev->pNextR3 = pNext;
1703	else
1704	pVM->pgm.s.pMmio2RangesR3 = pNext;
1705	pCur->pNextR3 = NULL;
1706
1707	/*
1708	* Free the memory.
1709	*/
1710	int rc2 = SUPR3PageFreeEx(pCur->pvR3, pCur->RamRange.cb >> PAGE_SHIFT);
1711	AssertRC(rc2);
1712	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
1713	rc = rc2;
1714
1715	uint32_t const cPages = pCur->RamRange.cb >> PAGE_SHIFT;
1716	rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pCur->RamRange.pszDesc);
1717	AssertRC(rc2);
1718	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
1719	rc = rc2;
1720
1721	/* we're leaking hyper memory here if done at runtime. */
1722	#ifdef VBOX_STRICT
1723	VMSTATE const enmState = VMR3GetState(pVM);
1724	AssertMsg( enmState == VMSTATE_POWERING_OFF
1725	\|\| enmState == VMSTATE_POWERING_OFF_LS
1726	\|\| enmState == VMSTATE_OFF
1727	\|\| enmState == VMSTATE_OFF_LS
1728	\|\| enmState == VMSTATE_DESTROYING
1729	\|\| enmState == VMSTATE_TERMINATED
1730	\|\| enmState == VMSTATE_CREATING
1731	, ("%s\n", VMR3GetStateName(enmState)));
1732	#endif
1733	/*rc = MMHyperFree(pVM, pCur);
1734	AssertRCReturn(rc, rc); - not safe, see the alloc call. */
1735
1736
1737	/* update page count stats */
1738	pVM->pgm.s.cAllPages -= cPages;
1739	pVM->pgm.s.cPrivatePages -= cPages;
1740
1741	/* next */
1742	pCur = pNext;
1743	}
1744	else
1745	{
1746	pPrev = pCur;
1747	pCur = pCur->pNextR3;
1748	}
1749	}
1750	pgmUnlock(pVM);
1751	return !cFound && iRegion != UINT32_MAX ? VERR_NOT_FOUND : rc;
1752	}
1753
1754
1755	/**
1756	* Maps a MMIO2 region.
1757	*
1758	* This is done when a guest / the bios / state loading changes the
1759	* PCI config. The replacing of base memory has the same restrictions
1760	* as during registration, of course.
1761	*
1762	* @returns VBox status code.
1763	*
1764	* @param pVM Pointer to the shared VM structure.
1765	* @param pDevIns The
1766	*/
1767	VMMR3DECL(int) PGMR3PhysMMIO2Map(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
1768	{
1769	/*
1770	* Validate input
1771	*/
1772	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1773	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
1774	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
1775	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
1776	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
1777	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
1778
1779	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
1780	AssertReturn(pCur, VERR_NOT_FOUND);
1781	AssertReturn(!pCur->fMapped, VERR_WRONG_ORDER);
1782	Assert(pCur->RamRange.GCPhys == NIL_RTGCPHYS);
1783	Assert(pCur->RamRange.GCPhysLast == NIL_RTGCPHYS);
1784
1785	const RTGCPHYS GCPhysLast = GCPhys + pCur->RamRange.cb - 1;
1786	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
1787
1788	/*
1789	* Find our location in the ram range list, checking for
1790	* restriction we don't bother implementing yet (partially overlapping).
1791	*/
1792	bool fRamExists = false;
1793	PPGMRAMRANGE pRamPrev = NULL;
1794	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
1795	while (pRam && GCPhysLast >= pRam->GCPhys)
1796	{
1797	if ( GCPhys <= pRam->GCPhysLast
1798	&& GCPhysLast >= pRam->GCPhys)
1799	{
1800	/* completely within? */
1801	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
1802	&& GCPhysLast <= pRam->GCPhysLast,
1803	("%RGp-%RGp (MMIO2/%s) falls partly outside %RGp-%RGp (%s)\n",
1804	GCPhys, GCPhysLast, pCur->RamRange.pszDesc,
1805	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
1806	VERR_PGM_RAM_CONFLICT);
1807	fRamExists = true;
1808	break;
1809	}
1810
1811	/* next */
1812	pRamPrev = pRam;
1813	pRam = pRam->pNextR3;
1814	}
1815	if (fRamExists)
1816	{
1817	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
1818	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
1819	while (cPagesLeft-- > 0)
1820	{
1821	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
1822	("%RGp isn't a RAM page (%d) - mapping %RGp-%RGp (MMIO2/%s).\n",
1823	GCPhys, PGM_PAGE_GET_TYPE(pPage), GCPhys, GCPhysLast, pCur->RamRange.pszDesc),
1824	VERR_PGM_RAM_CONFLICT);
1825	pPage++;
1826	}
1827	}
1828	Log(("PGMR3PhysMMIO2Map: %RGp-%RGp fRamExists=%RTbool %s\n",
1829	GCPhys, GCPhysLast, fRamExists, pCur->RamRange.pszDesc));
1830
1831	/*
1832	* Make the changes.
1833	*/
1834	pgmLock(pVM);
1835
1836	pCur->RamRange.GCPhys = GCPhys;
1837	pCur->RamRange.GCPhysLast = GCPhysLast;
1838	pCur->fMapped = true;
1839	pCur->fOverlapping = fRamExists;
1840
1841	if (fRamExists)
1842	{
1843	/** @todo use pgmR3PhysFreePageRange here. */
1844	uint32_t cPendingPages = 0;
1845	PGMMFREEPAGESREQ pReq;
1846	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1847	AssertLogRelRCReturn(rc, rc);
1848
1849	/* replace the pages, freeing all present RAM pages. */
1850	PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
1851	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
1852	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
1853	while (cPagesLeft-- > 0)
1854	{
1855	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
1856	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
1857
1858	RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
1859	PGM_PAGE_SET_HCPHYS(pPageDst, HCPhys);
1860	PGM_PAGE_SET_TYPE(pPageDst, PGMPAGETYPE_MMIO2);
1861	PGM_PAGE_SET_STATE(pPageDst, PGM_PAGE_STATE_ALLOCATED);
1862
1863	pVM->pgm.s.cZeroPages--;
1864	GCPhys += PAGE_SIZE;
1865	pPageSrc++;
1866	pPageDst++;
1867	}
1868
1869	if (cPendingPages)
1870	{
1871	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
1872	AssertLogRelRCReturn(rc, rc);
1873	}
1874	GMMR3FreePagesCleanup(pReq);
1875	pgmUnlock(pVM);
1876	}
1877	else
1878	{
1879	RTGCPHYS cb = pCur->RamRange.cb;
1880
1881	/* link in the ram range */
1882	pgmR3PhysLinkRamRange(pVM, &pCur->RamRange, pRamPrev);
1883	pgmUnlock(pVM);
1884
1885	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_MMIO2);
1886	}
1887
1888	return VINF_SUCCESS;
1889	}
1890
1891
1892	/**
1893	* Unmaps a MMIO2 region.
1894	*
1895	* This is done when a guest / the bios / state loading changes the
1896	* PCI config. The replacing of base memory has the same restrictions
1897	* as during registration, of course.
1898	*/
1899	VMMR3DECL(int) PGMR3PhysMMIO2Unmap(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
1900	{
1901	/*
1902	* Validate input
1903	*/
1904	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1905	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
1906	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
1907	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
1908	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
1909	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
1910
1911	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
1912	AssertReturn(pCur, VERR_NOT_FOUND);
1913	AssertReturn(pCur->fMapped, VERR_WRONG_ORDER);
1914	AssertReturn(pCur->RamRange.GCPhys == GCPhys, VERR_INVALID_PARAMETER);
1915	Assert(pCur->RamRange.GCPhysLast != NIL_RTGCPHYS);
1916
1917	Log(("PGMR3PhysMMIO2Unmap: %RGp-%RGp %s\n",
1918	pCur->RamRange.GCPhys, pCur->RamRange.GCPhysLast, pCur->RamRange.pszDesc));
1919
1920	/*
1921	* Unmap it.
1922	*/
1923	pgmLock(pVM);
1924
1925	RTGCPHYS GCPhysRangeREM;
1926	RTGCPHYS cbRangeREM;
1927	bool fInformREM;
1928	if (pCur->fOverlapping)
1929	{
1930	/* Restore the RAM pages we've replaced. */
1931	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
1932	while (pRam->GCPhys > pCur->RamRange.GCPhysLast)
1933	pRam = pRam->pNextR3;
1934
1935	RTHCPHYS const HCPhysZeroPg = pVM->pgm.s.HCPhysZeroPg;
1936	Assert(HCPhysZeroPg != 0 && HCPhysZeroPg != NIL_RTHCPHYS);
1937	PPGMPAGE pPageDst = &pRam->aPages[(pCur->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
1938	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
1939	while (cPagesLeft-- > 0)
1940	{
1941	PGM_PAGE_SET_HCPHYS(pPageDst, HCPhysZeroPg);
1942	PGM_PAGE_SET_TYPE(pPageDst, PGMPAGETYPE_RAM);
1943	PGM_PAGE_SET_STATE(pPageDst, PGM_PAGE_STATE_ZERO);
1944	PGM_PAGE_SET_PAGEID(pPageDst, NIL_GMM_PAGEID);
1945
1946	pVM->pgm.s.cZeroPages++;
1947	pPageDst++;
1948	}
1949
1950	GCPhysRangeREM = NIL_RTGCPHYS; /* shuts up gcc */
1951	cbRangeREM = RTGCPHYS_MAX; /* ditto */
1952	fInformREM = false;
1953	}
1954	else
1955	{
1956	GCPhysRangeREM = pCur->RamRange.GCPhys;
1957	cbRangeREM = pCur->RamRange.cb;
1958	fInformREM = true;
1959
1960	pgmR3PhysUnlinkRamRange(pVM, &pCur->RamRange);
1961	}
1962
1963	pCur->RamRange.GCPhys = NIL_RTGCPHYS;
1964	pCur->RamRange.GCPhysLast = NIL_RTGCPHYS;
1965	pCur->fOverlapping = false;
1966	pCur->fMapped = false;
1967
1968	pgmUnlock(pVM);
1969
1970	if (fInformREM)
1971	REMR3NotifyPhysRamDeregister(pVM, GCPhysRangeREM, cbRangeREM);
1972
1973	return VINF_SUCCESS;
1974	}
1975
1976
1977	/**
1978	* Checks if the given address is an MMIO2 base address or not.
1979	*
1980	* @returns true/false accordingly.
1981	* @param pVM Pointer to the shared VM structure.
1982	* @param pDevIns The owner of the memory, optional.
1983	* @param GCPhys The address to check.
1984	*/
1985	VMMR3DECL(bool) PGMR3PhysMMIO2IsBase(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
1986	{
1987	/*
1988	* Validate input
1989	*/
1990	VM_ASSERT_EMT_RETURN(pVM, false);
1991	AssertPtrReturn(pDevIns, false);
1992	AssertReturn(GCPhys != NIL_RTGCPHYS, false);
1993	AssertReturn(GCPhys != 0, false);
1994	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), false);
1995
1996	/*
1997	* Search the list.
1998	*/
1999	pgmLock(pVM);
2000	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2001	if (pCur->RamRange.GCPhys == GCPhys)
2002	{
2003	Assert(pCur->fMapped);
2004	pgmUnlock(pVM);
2005	return true;
2006	}
2007	pgmUnlock(pVM);
2008	return false;
2009	}
2010
2011
2012	/**
2013	* Gets the HC physical address of a page in the MMIO2 region.
2014	*
2015	* This is API is intended for MMHyper and shouldn't be called
2016	* by anyone else...
2017	*
2018	* @returns VBox status code.
2019	* @param pVM Pointer to the shared VM structure.
2020	* @param pDevIns The owner of the memory, optional.
2021	* @param iRegion The region.
2022	* @param off The page expressed an offset into the MMIO2 region.
2023	* @param pHCPhys Where to store the result.
2024	*/
2025	VMMR3DECL(int) PGMR3PhysMMIO2GetHCPhys(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, PRTHCPHYS pHCPhys)
2026	{
2027	/*
2028	* Validate input
2029	*/
2030	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2031	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2032	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2033
2034	pgmLock(pVM);
2035	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2036	AssertReturn(pCur, VERR_NOT_FOUND);
2037	AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2038
2039	PCPGMPAGE pPage = &pCur->RamRange.aPages[off >> PAGE_SHIFT];
2040	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2041	pgmUnlock(pVM);
2042	return VINF_SUCCESS;
2043	}
2044
2045
2046	/**
2047	* Maps a portion of an MMIO2 region into kernel space (host).
2048	*
2049	* The kernel mapping will become invalid when the MMIO2 memory is deregistered
2050	* or the VM is terminated.
2051	*
2052	* @return VBox status code.
2053	*
2054	* @param pVM Pointer to the shared VM structure.
2055	* @param pDevIns The device owning the MMIO2 memory.
2056	* @param iRegion The region.
2057	* @param off The offset into the region. Must be page aligned.
2058	* @param cb The number of bytes to map. Must be page aligned.
2059	* @param pszDesc Mapping description.
2060	* @param pR0Ptr Where to store the R0 address.
2061	*/
2062	VMMR3DECL(int) PGMR3PhysMMIO2MapKernel(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
2063	const char *pszDesc, PRTR0PTR pR0Ptr)
2064	{
2065	/*
2066	* Validate input.
2067	*/
2068	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2069	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2070	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2071
2072	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2073	AssertReturn(pCur, VERR_NOT_FOUND);
2074	AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2075	AssertReturn(cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2076	AssertReturn(off + cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2077
2078	/*
2079	* Pass the request on to the support library/driver.
2080	*/
2081	int rc = SUPR3PageMapKernel(pCur->pvR3, off, cb, 0, pR0Ptr);
2082
2083	return rc;
2084	}
2085
2086
2087	/**
2088	* Registers a ROM image.
2089	*
2090	* Shadowed ROM images requires double the amount of backing memory, so,
2091	* don't use that unless you have to. Shadowing of ROM images is process
2092	* where we can select where the reads go and where the writes go. On real
2093	* hardware the chipset provides means to configure this. We provide
2094	* PGMR3PhysProtectROM() for this purpose.
2095	*
2096	* A read-only copy of the ROM image will always be kept around while we
2097	* will allocate RAM pages for the changes on demand (unless all memory
2098	* is configured to be preallocated).
2099	*
2100	* @returns VBox status.
2101	* @param pVM VM Handle.
2102	* @param pDevIns The device instance owning the ROM.
2103	* @param GCPhys First physical address in the range.
2104	* Must be page aligned!
2105	* @param cbRange The size of the range (in bytes).
2106	* Must be page aligned!
2107	* @param pvBinary Pointer to the binary data backing the ROM image.
2108	* This must be exactly \a cbRange in size.
2109	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
2110	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
2111	* @param pszDesc Pointer to description string. This must not be freed.
2112	*
2113	* @remark There is no way to remove the rom, automatically on device cleanup or
2114	* manually from the device yet. This isn't difficult in any way, it's
2115	* just not something we expect to be necessary for a while.
2116	*/
2117	VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
2118	const void pvBinary, uint32_t fFlags, const char pszDesc)
2119	{
2120	Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p fFlags=%#x pszDesc=%s\n",
2121	pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, fFlags, pszDesc));
2122
2123	/*
2124	* Validate input.
2125	*/
2126	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2127	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
2128	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
2129	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2130	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2131	AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
2132	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2133	AssertReturn(!(fFlags & ~(PGMPHYS_ROM_FLAGS_SHADOWED \| PGMPHYS_ROM_FLAGS_PERMANENT_BINARY)), VERR_INVALID_PARAMETER);
2134	VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
2135
2136	const uint32_t cPages = cb >> PAGE_SHIFT;
2137
2138	/*
2139	* Find the ROM location in the ROM list first.
2140	*/
2141	PPGMROMRANGE pRomPrev = NULL;
2142	PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3;
2143	while (pRom && GCPhysLast >= pRom->GCPhys)
2144	{
2145	if ( GCPhys <= pRom->GCPhysLast
2146	&& GCPhysLast >= pRom->GCPhys)
2147	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
2148	GCPhys, GCPhysLast, pszDesc,
2149	pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
2150	VERR_PGM_RAM_CONFLICT);
2151	/* next */
2152	pRomPrev = pRom;
2153	pRom = pRom->pNextR3;
2154	}
2155
2156	/*
2157	* Find the RAM location and check for conflicts.
2158	*
2159	* Conflict detection is a bit different than for RAM
2160	* registration since a ROM can be located within a RAM
2161	* range. So, what we have to check for is other memory
2162	* types (other than RAM that is) and that we don't span
2163	* more than one RAM range (layz).
2164	*/
2165	bool fRamExists = false;
2166	PPGMRAMRANGE pRamPrev = NULL;
2167	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
2168	while (pRam && GCPhysLast >= pRam->GCPhys)
2169	{
2170	if ( GCPhys <= pRam->GCPhysLast
2171	&& GCPhysLast >= pRam->GCPhys)
2172	{
2173	/* completely within? */
2174	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
2175	&& GCPhysLast <= pRam->GCPhysLast,
2176	("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
2177	GCPhys, GCPhysLast, pszDesc,
2178	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2179	VERR_PGM_RAM_CONFLICT);
2180	fRamExists = true;
2181	break;
2182	}
2183
2184	/* next */
2185	pRamPrev = pRam;
2186	pRam = pRam->pNextR3;
2187	}
2188	if (fRamExists)
2189	{
2190	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2191	uint32_t cPagesLeft = cPages;
2192	while (cPagesLeft-- > 0)
2193	{
2194	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
2195	("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
2196	pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << PAGE_SHIFT),
2197	pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
2198	Assert(PGM_PAGE_IS_ZERO(pPage));
2199	pPage++;
2200	}
2201	}
2202
2203	/*
2204	* Update the base memory reservation if necessary.
2205	*/
2206	uint32_t cExtraBaseCost = fRamExists ? cPages : 0;
2207	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
2208	cExtraBaseCost += cPages;
2209	if (cExtraBaseCost)
2210	{
2211	int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
2212	if (RT_FAILURE(rc))
2213	return rc;
2214	}
2215
2216	/*
2217	* Allocate memory for the virgin copy of the RAM.
2218	*/
2219	PGMMALLOCATEPAGESREQ pReq;
2220	int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
2221	AssertRCReturn(rc, rc);
2222
2223	for (uint32_t iPage = 0; iPage < cPages; iPage++)
2224	{
2225	pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
2226	pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
2227	pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
2228	}
2229
2230	pgmLock(pVM);
2231	rc = GMMR3AllocatePagesPerform(pVM, pReq);
2232	pgmUnlock(pVM);
2233	if (RT_FAILURE(rc))
2234	{
2235	GMMR3AllocatePagesCleanup(pReq);
2236	return rc;
2237	}
2238
2239	/*
2240	* Allocate the new ROM range and RAM range (if necessary).
2241	*/
2242	PPGMROMRANGE pRomNew;
2243	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
2244	if (RT_SUCCESS(rc))
2245	{
2246	PPGMRAMRANGE pRamNew = NULL;
2247	if (!fRamExists)
2248	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), sizeof(PGMPAGE), MM_TAG_PGM_PHYS, (void **)&pRamNew);
2249	if (RT_SUCCESS(rc))
2250	{
2251	pgmLock(pVM);
2252
2253	/*
2254	* Initialize and insert the RAM range (if required).
2255	*/
2256	PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
2257	if (!fRamExists)
2258	{
2259	pRamNew->pSelfR0 = MMHyperCCToR0(pVM, pRamNew);
2260	pRamNew->pSelfRC = MMHyperCCToRC(pVM, pRamNew);
2261	pRamNew->GCPhys = GCPhys;
2262	pRamNew->GCPhysLast = GCPhysLast;
2263	pRamNew->cb = cb;
2264	pRamNew->pszDesc = pszDesc;
2265	pRamNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_ROM;
2266	pRamNew->pvR3 = NULL;
2267	pRamNew->paLSPages = NULL;
2268
2269	PPGMPAGE pPage = &pRamNew->aPages[0];
2270	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
2271	{
2272	PGM_PAGE_INIT(pPage,
2273	pReq->aPages[iPage].HCPhysGCPhys,
2274	pReq->aPages[iPage].idPage,
2275	PGMPAGETYPE_ROM,
2276	PGM_PAGE_STATE_ALLOCATED);
2277
2278	pRomPage->Virgin = *pPage;
2279	}
2280
2281	pVM->pgm.s.cAllPages += cPages;
2282	pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
2283	}
2284	else
2285	{
2286	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2287	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
2288	{
2289	PGM_PAGE_SET_TYPE(pPage, PGMPAGETYPE_ROM);
2290	PGM_PAGE_SET_HCPHYS(pPage, pReq->aPages[iPage].HCPhysGCPhys);
2291	PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
2292	PGM_PAGE_SET_PAGEID(pPage, pReq->aPages[iPage].idPage);
2293
2294	pRomPage->Virgin = *pPage;
2295	}
2296
2297	pRamNew = pRam;
2298
2299	pVM->pgm.s.cZeroPages -= cPages;
2300	}
2301	pVM->pgm.s.cPrivatePages += cPages;
2302
2303	pgmUnlock(pVM);
2304
2305
2306	/*
2307	* !HACK ALERT! REM + (Shadowed) ROM ==> mess.
2308	*
2309	* If it's shadowed we'll register the handler after the ROM notification
2310	* so we get the access handler callbacks that we should. If it isn't
2311	* shadowed we'll do it the other way around to make REM use the built-in
2312	* ROM behavior and not the handler behavior (which is to route all access
2313	* to PGM atm).
2314	*/
2315	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
2316	{
2317	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, true /* fShadowed */);
2318	rc = PGMR3HandlerPhysicalRegister(pVM,
2319	fFlags & PGMPHYS_ROM_FLAGS_SHADOWED
2320	? PGMPHYSHANDLERTYPE_PHYSICAL_ALL
2321	: PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
2322	GCPhys, GCPhysLast,
2323	pgmR3PhysRomWriteHandler, pRomNew,
2324	NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
2325	NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
2326	}
2327	else
2328	{
2329	rc = PGMR3HandlerPhysicalRegister(pVM,
2330	fFlags & PGMPHYS_ROM_FLAGS_SHADOWED
2331	? PGMPHYSHANDLERTYPE_PHYSICAL_ALL
2332	: PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
2333	GCPhys, GCPhysLast,
2334	pgmR3PhysRomWriteHandler, pRomNew,
2335	NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
2336	NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
2337	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, false /* fShadowed */);
2338	}
2339	if (RT_SUCCESS(rc))
2340	{
2341	pgmLock(pVM);
2342
2343	/*
2344	* Copy the image over to the virgin pages.
2345	* This must be done after linking in the RAM range.
2346	*/
2347	PPGMPAGE pRamPage = &pRamNew->aPages[(GCPhys - pRamNew->GCPhys) >> PAGE_SHIFT];
2348	for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
2349	{
2350	void *pvDstPage;
2351	rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << PAGE_SHIFT), &pvDstPage);
2352	if (RT_FAILURE(rc))
2353	{
2354	VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
2355	break;
2356	}
2357	memcpy(pvDstPage, (const uint8_t *)pvBinary + (iPage << PAGE_SHIFT), PAGE_SIZE);
2358	}
2359	if (RT_SUCCESS(rc))
2360	{
2361	/*
2362	* Initialize the ROM range.
2363	* Note that the Virgin member of the pages has already been initialized above.
2364	*/
2365	pRomNew->GCPhys = GCPhys;
2366	pRomNew->GCPhysLast = GCPhysLast;
2367	pRomNew->cb = cb;
2368	pRomNew->fFlags = fFlags;
2369	pRomNew->idSavedState = UINT8_MAX;
2370	pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY ? pvBinary : NULL;
2371	pRomNew->pszDesc = pszDesc;
2372
2373	for (unsigned iPage = 0; iPage < cPages; iPage++)
2374	{
2375	PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
2376	pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
2377	PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
2378	}
2379
2380	/* update the page count stats */
2381	pVM->pgm.s.cZeroPages += cPages;
2382	pVM->pgm.s.cAllPages += cPages;
2383
2384	/*
2385	* Insert the ROM range, tell REM and return successfully.
2386	*/
2387	pRomNew->pNextR3 = pRom;
2388	pRomNew->pNextR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
2389	pRomNew->pNextRC = pRom ? MMHyperCCToRC(pVM, pRom) : NIL_RTRCPTR;
2390
2391	if (pRomPrev)
2392	{
2393	pRomPrev->pNextR3 = pRomNew;
2394	pRomPrev->pNextR0 = MMHyperCCToR0(pVM, pRomNew);
2395	pRomPrev->pNextRC = MMHyperCCToRC(pVM, pRomNew);
2396	}
2397	else
2398	{
2399	pVM->pgm.s.pRomRangesR3 = pRomNew;
2400	pVM->pgm.s.pRomRangesR0 = MMHyperCCToR0(pVM, pRomNew);
2401	pVM->pgm.s.pRomRangesRC = MMHyperCCToRC(pVM, pRomNew);
2402	}
2403
2404	GMMR3AllocatePagesCleanup(pReq);
2405	pgmUnlock(pVM);
2406	return VINF_SUCCESS;
2407	}
2408
2409	/* bail out */
2410
2411	pgmUnlock(pVM);
2412	int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
2413	AssertRC(rc2);
2414	pgmLock(pVM);
2415	}
2416
2417	if (!fRamExists)
2418	{
2419	pgmR3PhysUnlinkRamRange2(pVM, pRamNew, pRamPrev);
2420	MMHyperFree(pVM, pRamNew);
2421	}
2422	}
2423	MMHyperFree(pVM, pRomNew);
2424	}
2425
2426	/** @todo Purge the mapping cache or something... */
2427	GMMR3FreeAllocatedPages(pVM, pReq);
2428	GMMR3AllocatePagesCleanup(pReq);
2429	pgmUnlock(pVM);
2430	return rc;
2431	}
2432
2433
2434	/**
2435	* \#PF Handler callback for ROM write accesses.
2436	*
2437	* @returns VINF_SUCCESS if the handler have carried out the operation.
2438	* @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
2439	* @param pVM VM Handle.
2440	* @param GCPhys The physical address the guest is writing to.
2441	* @param pvPhys The HC mapping of that address.
2442	* @param pvBuf What the guest is reading/writing.
2443	* @param cbBuf How much it's reading/writing.
2444	* @param enmAccessType The access type.
2445	* @param pvUser User argument.
2446	*/
2447	static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser)
2448	{
2449	PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
2450	const uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
2451	Assert(iPage < (pRom->cb >> PAGE_SHIFT));
2452	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
2453	Log5(("pgmR3PhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
2454
2455	if (enmAccessType == PGMACCESSTYPE_READ)
2456	{
2457	switch (pRomPage->enmProt)
2458	{
2459	/*
2460	* Take the default action.
2461	*/
2462	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
2463	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
2464	case PGMROMPROT_READ_ROM_WRITE_RAM:
2465	case PGMROMPROT_READ_RAM_WRITE_RAM:
2466	return VINF_PGM_HANDLER_DO_DEFAULT;
2467
2468	default:
2469	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
2470	pRom->aPages[iPage].enmProt, iPage, GCPhys),
2471	VERR_INTERNAL_ERROR);
2472	}
2473	}
2474	else
2475	{
2476	Assert(enmAccessType == PGMACCESSTYPE_WRITE);
2477	switch (pRomPage->enmProt)
2478	{
2479	/*
2480	* Ignore writes.
2481	*/
2482	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
2483	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
2484	return VINF_SUCCESS;
2485
2486	/*
2487	* Write to the ram page.
2488	*/
2489	case PGMROMPROT_READ_ROM_WRITE_RAM:
2490	case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's way simpler that way. */
2491	{
2492	/* This should be impossible now, pvPhys doesn't work cross page anylonger. */
2493	Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> PAGE_SHIFT) == iPage);
2494
2495	/*
2496	* Take the lock, do lazy allocation, map the page and copy the data.
2497	*
2498	* Note that we have to bypass the mapping TLB since it works on
2499	* guest physical addresses and entering the shadow page would
2500	* kind of screw things up...
2501	*/
2502	int rc = pgmLock(pVM);
2503	AssertRC(rc);
2504
2505	PPGMPAGE pShadowPage = &pRomPage->Shadow;
2506	if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
2507	{
2508	pShadowPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
2509	AssertLogRelReturn(pShadowPage, VERR_INTERNAL_ERROR);
2510	}
2511
2512	void *pvDstPage;
2513	rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
2514	if (RT_SUCCESS(rc))
2515	{
2516	memcpy((uint8_t *)pvDstPage + (GCPhys & PAGE_OFFSET_MASK), pvBuf, cbBuf);
2517	pRomPage->LiveSave.fWrittenTo = true;
2518	}
2519
2520	pgmUnlock(pVM);
2521	return rc;
2522	}
2523
2524	default:
2525	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
2526	pRom->aPages[iPage].enmProt, iPage, GCPhys),
2527	VERR_INTERNAL_ERROR);
2528	}
2529	}
2530	}
2531
2532
2533	/**
2534	* Called by PGMR3Reset to reset the shadow, switch to the virgin,
2535	* and verify that the virgin part is untouched.
2536	*
2537	* This is done after the normal memory has been cleared.
2538	*
2539	* ASSUMES that the caller owns the PGM lock.
2540	*
2541	* @param pVM The VM handle.
2542	*/
2543	int pgmR3PhysRomReset(PVM pVM)
2544	{
2545	Assert(PGMIsLockOwner(pVM));
2546	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
2547	{
2548	const uint32_t cPages = pRom->cb >> PAGE_SHIFT;
2549
2550	if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
2551	{
2552	/*
2553	* Reset the physical handler.
2554	*/
2555	int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
2556	AssertRCReturn(rc, rc);
2557
2558	/*
2559	* What we do with the shadow pages depends on the memory
2560	* preallocation option. If not enabled, we'll just throw
2561	* out all the dirty pages and replace them by the zero page.
2562	*/
2563	if (!pVM->pgm.s.fRamPreAlloc)
2564	{
2565	/* Free the dirty pages. */
2566	uint32_t cPendingPages = 0;
2567	PGMMFREEPAGESREQ pReq;
2568	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2569	AssertRCReturn(rc, rc);
2570
2571	for (uint32_t iPage = 0; iPage < cPages; iPage++)
2572	if (PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) != PGM_PAGE_STATE_ZERO)
2573	{
2574	Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
2575	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow, pRom->GCPhys + (iPage << PAGE_SHIFT));
2576	AssertLogRelRCReturn(rc, rc);
2577	}
2578
2579	if (cPendingPages)
2580	{
2581	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2582	AssertLogRelRCReturn(rc, rc);
2583	}
2584	GMMR3FreePagesCleanup(pReq);
2585	}
2586	else
2587	{
2588	/* clear all the shadow pages. */
2589	for (uint32_t iPage = 0; iPage < cPages; iPage++)
2590	{
2591	Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) != PGM_PAGE_STATE_ZERO);
2592	void *pvDstPage;
2593	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
2594	rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
2595	if (RT_FAILURE(rc))
2596	break;
2597	ASMMemZeroPage(pvDstPage);
2598	}
2599	AssertRCReturn(rc, rc);
2600	}
2601	}
2602
2603	#ifdef VBOX_STRICT
2604	/*
2605	* Verify that the virgin page is unchanged if possible.
2606	*/
2607	if (pRom->pvOriginal)
2608	{
2609	uint8_t const pbSrcPage = (uint8_t const )pRom->pvOriginal;
2610	for (uint32_t iPage = 0; iPage < cPages; iPage++, pbSrcPage += PAGE_SIZE)
2611	{
2612	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
2613	void const *pvDstPage;
2614	int rc = pgmPhysPageMapReadOnly(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPage);
2615	if (RT_FAILURE(rc))
2616	break;
2617	if (memcmp(pvDstPage, pbSrcPage, PAGE_SIZE))
2618	LogRel(("pgmR3PhysRomReset: %RGp rom page changed (%s) - loaded saved state?\n",
2619	GCPhys, pRom->pszDesc));
2620	}
2621	}
2622	#endif
2623	}
2624
2625	return VINF_SUCCESS;
2626	}
2627
2628
2629	/**
2630	* Change the shadowing of a range of ROM pages.
2631	*
2632	* This is intended for implementing chipset specific memory registers
2633	* and will not be very strict about the input. It will silently ignore
2634	* any pages that are not the part of a shadowed ROM.
2635	*
2636	* @returns VBox status code.
2637	* @retval VINF_PGM_SYNC_CR3
2638	*
2639	* @param pVM Pointer to the shared VM structure.
2640	* @param GCPhys Where to start. Page aligned.
2641	* @param cb How much to change. Page aligned.
2642	* @param enmProt The new ROM protection.
2643	*/
2644	VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
2645	{
2646	/*
2647	* Check input
2648	*/
2649	if (!cb)
2650	return VINF_SUCCESS;
2651	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2652	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2653	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2654	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2655	AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
2656
2657	/*
2658	* Process the request.
2659	*/
2660	pgmLock(pVM);
2661	int rc = VINF_SUCCESS;
2662	bool fFlushTLB = false;
2663	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
2664	{
2665	if ( GCPhys <= pRom->GCPhysLast
2666	&& GCPhysLast >= pRom->GCPhys
2667	&& (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
2668	{
2669	/*
2670	* Iterate the relevant pages and make necessary the changes.
2671	*/
2672	bool fChanges = false;
2673	uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
2674	? pRom->cb >> PAGE_SHIFT
2675	: (GCPhysLast - pRom->GCPhys + 1) >> PAGE_SHIFT;
2676	for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
2677	iPage < cPages;
2678	iPage++)
2679	{
2680	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
2681	if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
2682	{
2683	fChanges = true;
2684
2685	/* flush references to the page. */
2686	PPGMPAGE pRamPage = pgmPhysGetPage(&pVM->pgm.s, pRom->GCPhys + (iPage << PAGE_SHIFT));
2687	int rc2 = pgmPoolTrackFlushGCPhys(pVM, pRamPage, &fFlushTLB);
2688	if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS \|\| RT_FAILURE(rc2)))
2689	rc = rc2;
2690
2691	PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
2692	PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
2693
2694	pOld = pRamPage;
2695	pRamPage = pNew;
2696	/** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
2697	}
2698	pRomPage->enmProt = enmProt;
2699	}
2700
2701	/*
2702	* Reset the access handler if we made changes, no need
2703	* to optimize this.
2704	*/
2705	if (fChanges)
2706	{
2707	int rc = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
2708	if (RT_FAILURE(rc))
2709	{
2710	pgmUnlock(pVM);
2711	AssertRC(rc);
2712	return rc;
2713	}
2714	}
2715
2716	/* Advance - cb isn't updated. */
2717	GCPhys = pRom->GCPhys + (cPages << PAGE_SHIFT);
2718	}
2719	}
2720	pgmUnlock(pVM);
2721	if (fFlushTLB)
2722	PGM_INVL_ALL_VCPU_TLBS(pVM);
2723
2724	return rc;
2725	}
2726
2727
2728	/**
2729	* Sets the Address Gate 20 state.
2730	*
2731	* @param pVCpu The VCPU to operate on.
2732	* @param fEnable True if the gate should be enabled.
2733	* False if the gate should be disabled.
2734	*/
2735	VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
2736	{
2737	LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
2738	if (pVCpu->pgm.s.fA20Enabled != fEnable)
2739	{
2740	pVCpu->pgm.s.fA20Enabled = fEnable;
2741	pVCpu->pgm.s.GCPhysA20Mask = ~(RTGCPHYS)(!fEnable << 20);
2742	REMR3A20Set(pVCpu->pVMR3, pVCpu, fEnable);
2743	/** @todo we're not handling this correctly for VT-x / AMD-V. See #2911 */
2744	}
2745	}
2746
2747
2748	/**
2749	* Tree enumeration callback for dealing with age rollover.
2750	* It will perform a simple compression of the current age.
2751	*/
2752	static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
2753	{
2754	Assert(PGMIsLockOwner((PVM)pvUser));
2755	/* Age compression - ASSUMES iNow == 4. */
2756	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
2757	if (pChunk->iAge >= UINT32_C(0xffffff00))
2758	pChunk->iAge = 3;
2759	else if (pChunk->iAge >= UINT32_C(0xfffff000))
2760	pChunk->iAge = 2;
2761	else if (pChunk->iAge)
2762	pChunk->iAge = 1;
2763	else /* iAge = 0 */
2764	pChunk->iAge = 4;
2765
2766	/* reinsert */
2767	PVM pVM = (PVM)pvUser;
2768	RTAvllU32Remove(&pVM->pgm.s.ChunkR3Map.pAgeTree, pChunk->AgeCore.Key);
2769	pChunk->AgeCore.Key = pChunk->iAge;
2770	RTAvllU32Insert(&pVM->pgm.s.ChunkR3Map.pAgeTree, &pChunk->AgeCore);
2771	return 0;
2772	}
2773
2774
2775	/**
2776	* Tree enumeration callback that updates the chunks that have
2777	* been used since the last
2778	*/
2779	static DECLCALLBACK(int) pgmR3PhysChunkAgeingCallback(PAVLU32NODECORE pNode, void *pvUser)
2780	{
2781	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
2782	if (!pChunk->iAge)
2783	{
2784	PVM pVM = (PVM)pvUser;
2785	RTAvllU32Remove(&pVM->pgm.s.ChunkR3Map.pAgeTree, pChunk->AgeCore.Key);
2786	pChunk->AgeCore.Key = pChunk->iAge = pVM->pgm.s.ChunkR3Map.iNow;
2787	RTAvllU32Insert(&pVM->pgm.s.ChunkR3Map.pAgeTree, &pChunk->AgeCore);
2788	}
2789
2790	return 0;
2791	}
2792
2793
2794	/**
2795	* Performs ageing of the ring-3 chunk mappings.
2796	*
2797	* @param pVM The VM handle.
2798	*/
2799	VMMR3DECL(void) PGMR3PhysChunkAgeing(PVM pVM)
2800	{
2801	pgmLock(pVM);
2802	pVM->pgm.s.ChunkR3Map.AgeingCountdown = RT_MIN(pVM->pgm.s.ChunkR3Map.cMax / 4, 1024);
2803	pVM->pgm.s.ChunkR3Map.iNow++;
2804	if (pVM->pgm.s.ChunkR3Map.iNow == 0)
2805	{
2806	pVM->pgm.s.ChunkR3Map.iNow = 4;
2807	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkAgeingRolloverCallback, pVM);
2808	}
2809	else
2810	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkAgeingCallback, pVM);
2811	pgmUnlock(pVM);
2812	}
2813
2814
2815	/**
2816	* The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
2817	*/
2818	typedef struct PGMR3PHYSCHUNKUNMAPCB
2819	{
2820	PVM pVM; /*< The VM handle. /
2821	PPGMCHUNKR3MAP pChunk; /*< The chunk to unmap. /
2822	} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
2823
2824
2825	/**
2826	* Callback used to find the mapping that's been unused for
2827	* the longest time.
2828	*/
2829	static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLLU32NODECORE pNode, void *pvUser)
2830	{
2831	do
2832	{
2833	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)((uint8_t *)pNode - RT_OFFSETOF(PGMCHUNKR3MAP, AgeCore));
2834	if ( pChunk->iAge
2835	&& !pChunk->cRefs)
2836	{
2837	/*
2838	* Check that it's not in any of the TLBs.
2839	*/
2840	PVM pVM = ((PPGMR3PHYSCHUNKUNMAPCB)pvUser)->pVM;
2841	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
2842	if (pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk == pChunk)
2843	{
2844	pChunk = NULL;
2845	break;
2846	}
2847	if (pChunk)
2848	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
2849	if (pVM->pgm.s.PhysTlbHC.aEntries[i].pMap == pChunk)
2850	{
2851	pChunk = NULL;
2852	break;
2853	}
2854	if (pChunk)
2855	{
2856	((PPGMR3PHYSCHUNKUNMAPCB)pvUser)->pChunk = pChunk;
2857	return 1; /* done */
2858	}
2859	}
2860
2861	/* next with the same age - this version of the AVL API doesn't enumerate the list, so we have to do it. */
2862	pNode = pNode->pList;
2863	} while (pNode);
2864	return 0;
2865	}
2866
2867
2868	/**
2869	* Finds a good candidate for unmapping when the ring-3 mapping cache is full.
2870	*
2871	* The candidate will not be part of any TLBs, so no need to flush
2872	* anything afterwards.
2873	*
2874	* @returns Chunk id.
2875	* @param pVM The VM handle.
2876	*/
2877	static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
2878	{
2879	Assert(PGMIsLockOwner(pVM));
2880
2881	/*
2882	* Do tree ageing first?
2883	*/
2884	if (pVM->pgm.s.ChunkR3Map.AgeingCountdown-- == 0)
2885	PGMR3PhysChunkAgeing(pVM);
2886
2887	/*
2888	* Enumerate the age tree starting with the left most node.
2889	*/
2890	PGMR3PHYSCHUNKUNMAPCB Args;
2891	Args.pVM = pVM;
2892	Args.pChunk = NULL;
2893	if (RTAvllU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pAgeTree, true /fFromLeft/, pgmR3PhysChunkUnmapCandidateCallback, pVM))
2894	return Args.pChunk->Core.Key;
2895	return INT32_MAX;
2896	}
2897
2898
2899	/**
2900	* Maps the given chunk into the ring-3 mapping cache.
2901	*
2902	* This will call ring-0.
2903	*
2904	* @returns VBox status code.
2905	* @param pVM The VM handle.
2906	* @param idChunk The chunk in question.
2907	* @param ppChunk Where to store the chunk tracking structure.
2908	*
2909	* @remarks Called from within the PGM critical section.
2910	*/
2911	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
2912	{
2913	int rc;
2914
2915	Assert(PGMIsLockOwner(pVM));
2916	/*
2917	* Allocate a new tracking structure first.
2918	*/
2919	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2920	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAlloc(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
2921	#else
2922	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3UkHeapAlloc(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk), NULL);
2923	#endif
2924	AssertReturn(pChunk, VERR_NO_MEMORY);
2925	pChunk->Core.Key = idChunk;
2926	pChunk->AgeCore.Key = pVM->pgm.s.ChunkR3Map.iNow;
2927	pChunk->iAge = 0;
2928	pChunk->cRefs = 0;
2929	pChunk->cPermRefs = 0;
2930	pChunk->pv = NULL;
2931
2932	/*
2933	* Request the ring-0 part to map the chunk in question and if
2934	* necessary unmap another one to make space in the mapping cache.
2935	*/
2936	GMMMAPUNMAPCHUNKREQ Req;
2937	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
2938	Req.Hdr.cbReq = sizeof(Req);
2939	Req.pvR3 = NULL;
2940	Req.idChunkMap = idChunk;
2941	Req.idChunkUnmap = NIL_GMM_CHUNKID;
2942	if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
2943	Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
2944	/** @todo This is wrong. Any thread in the VM process should be able to do this,
2945	* there are depenenecies on this. What currently saves the day is that
2946	* we don't unmap anything and that all non-zero memory will therefore
2947	* be present when non-EMTs tries to access it. */
2948	rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
2949	if (RT_SUCCESS(rc))
2950	{
2951	/*
2952	* Update the tree.
2953	*/
2954	/* insert the new one. */
2955	AssertPtr(Req.pvR3);
2956	pChunk->pv = Req.pvR3;
2957	bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
2958	AssertRelease(fRc);
2959	pVM->pgm.s.ChunkR3Map.c++;
2960
2961	fRc = RTAvllU32Insert(&pVM->pgm.s.ChunkR3Map.pAgeTree, &pChunk->AgeCore);
2962	AssertRelease(fRc);
2963
2964	/* remove the unmapped one. */
2965	if (Req.idChunkUnmap != NIL_GMM_CHUNKID)
2966	{
2967	PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
2968	AssertRelease(pUnmappedChunk);
2969	pUnmappedChunk->pv = NULL;
2970	pUnmappedChunk->Core.Key = UINT32_MAX;
2971	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2972	MMR3HeapFree(pUnmappedChunk);
2973	#else
2974	MMR3UkHeapFree(pVM, pUnmappedChunk, MM_TAG_PGM_CHUNK_MAPPING);
2975	#endif
2976	pVM->pgm.s.ChunkR3Map.c--;
2977	}
2978	}
2979	else
2980	{
2981	AssertRC(rc);
2982	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2983	MMR3HeapFree(pChunk);
2984	#else
2985	MMR3UkHeapFree(pVM, pChunk, MM_TAG_PGM_CHUNK_MAPPING);
2986	#endif
2987	pChunk = NULL;
2988	}
2989
2990	*ppChunk = pChunk;
2991	return rc;
2992	}
2993
2994
2995	/**
2996	* For VMMCALLRING3_PGM_MAP_CHUNK, considered internal.
2997	*
2998	* @returns see pgmR3PhysChunkMap.
2999	* @param pVM The VM handle.
3000	* @param idChunk The chunk to map.
3001	*/
3002	VMMR3DECL(int) PGMR3PhysChunkMap(PVM pVM, uint32_t idChunk)
3003	{
3004	PPGMCHUNKR3MAP pChunk;
3005	int rc;
3006
3007	pgmLock(pVM);
3008	rc = pgmR3PhysChunkMap(pVM, idChunk, &pChunk);
3009	pgmUnlock(pVM);
3010	return rc;
3011	}
3012
3013
3014	/**
3015	* Invalidates the TLB for the ring-3 mapping cache.
3016	*
3017	* @param pVM The VM handle.
3018	*/
3019	VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
3020	{
3021	pgmLock(pVM);
3022	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
3023	{
3024	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
3025	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
3026	}
3027	pgmUnlock(pVM);
3028	}
3029
3030
3031	/**
3032	* Response to VM_FF_PGM_NEED_HANDY_PAGES and VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES.
3033	*
3034	* This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
3035	* signal and clear the out of memory condition. When contracted, this API is
3036	* used to try clear the condition when the user wants to resume.
3037	*
3038	* @returns The following VBox status codes.
3039	* @retval VINF_SUCCESS on success. FFs cleared.
3040	* @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
3041	* this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
3042	*
3043	* @param pVM The VM handle.
3044	*
3045	* @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
3046	* in EM.cpp and shouldn't be propagated outside TRPM, HWACCM, EM and
3047	* pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
3048	* handler.
3049	*/
3050	VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
3051	{
3052	pgmLock(pVM);
3053
3054	/*
3055	* Allocate more pages, noting down the index of the first new page.
3056	*/
3057	uint32_t iClear = pVM->pgm.s.cHandyPages;
3058	AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_INTERNAL_ERROR);
3059	Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
3060	int rcAlloc = VINF_SUCCESS;
3061	int rcSeed = VINF_SUCCESS;
3062	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
3063	while (rc == VERR_GMM_SEED_ME)
3064	{
3065	void *pvChunk;
3066	rcAlloc = rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
3067	if (RT_SUCCESS(rc))
3068	{
3069	rcSeed = rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
3070	if (RT_FAILURE(rc))
3071	SUPR3PageFree(pvChunk, GMM_CHUNK_SIZE >> PAGE_SHIFT);
3072	}
3073	if (RT_SUCCESS(rc))
3074	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
3075	}
3076
3077	if (RT_SUCCESS(rc))
3078	{
3079	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
3080	Assert(pVM->pgm.s.cHandyPages > 0);
3081	VM_FF_CLEAR(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
3082	VM_FF_CLEAR(pVM, VM_FF_PGM_NO_MEMORY);
3083
3084	/*
3085	* Clear the pages.
3086	*/
3087	while (iClear < pVM->pgm.s.cHandyPages)
3088	{
3089	PGMMPAGEDESC pPage = &pVM->pgm.s.aHandyPages[iClear];
3090	void *pv;
3091	rc = pgmPhysPageMapByPageID(pVM, pPage->idPage, pPage->HCPhysGCPhys, &pv);
3092	AssertLogRelMsgBreak(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc", pPage->idPage, pPage->HCPhysGCPhys, rc));
3093	ASMMemZeroPage(pv);
3094	iClear++;
3095	Log3(("PGMR3PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
3096	}
3097	}
3098	else
3099	{
3100	/*
3101	* We should never get here unless there is a genuine shortage of
3102	* memory (or some internal error). Flag the error so the VM can be
3103	* suspended ASAP and the user informed. If we're totally out of
3104	* handy pages we will return failure.
3105	*/
3106	/* Report the failure. */
3107	LogRel(("PGM: Failed to procure handy pages; rc=%Rrc rcAlloc=%Rrc rcSeed=%Rrc cHandyPages=%#x\n"
3108	" cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
3109	rc, rcAlloc, rcSeed,
3110	pVM->pgm.s.cHandyPages,
3111	pVM->pgm.s.cAllPages,
3112	pVM->pgm.s.cPrivatePages,
3113	pVM->pgm.s.cSharedPages,
3114	pVM->pgm.s.cZeroPages));
3115	if ( rc != VERR_NO_MEMORY
3116	&& rc != VERR_LOCK_FAILED)
3117	{
3118	for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
3119	{
3120	LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
3121	i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
3122	pVM->pgm.s.aHandyPages[i].idSharedPage));
3123	uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
3124	if (idPage != NIL_GMM_PAGEID)
3125	{
3126	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
3127	pRam;
3128	pRam = pRam->pNextR3)
3129	{
3130	uint32_t const cPages = pRam->cb >> PAGE_SHIFT;
3131	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3132	if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
3133	LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
3134	pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
3135	}
3136	}
3137	}
3138	}
3139
3140	/* Set the FFs and adjust rc. */
3141	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
3142	VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
3143	if ( rc == VERR_NO_MEMORY
3144	\|\| rc == VERR_LOCK_FAILED)
3145	rc = VINF_EM_NO_MEMORY;
3146	}
3147
3148	pgmUnlock(pVM);
3149	return rc;
3150	}
3151
3152
3153	/**
3154	* Frees the specified RAM page and replaces it with the ZERO page.
3155	*
3156	* This is used by ballooning, remapping MMIO2 and RAM reset.
3157	*
3158	* @param pVM Pointer to the shared VM structure.
3159	* @param pReq Pointer to the request.
3160	* @param pPage Pointer to the page structure.
3161	* @param GCPhys The guest physical address of the page, if applicable.
3162	*
3163	* @remarks The caller must own the PGM lock.
3164	*/
3165	static int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys)
3166	{
3167	/*
3168	* Assert sanity.
3169	*/
3170	Assert(PGMIsLockOwner(pVM));
3171	if (RT_UNLIKELY( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
3172	&& PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
3173	{
3174	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
3175	return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
3176	}
3177
3178	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ZERO)
3179	return VINF_SUCCESS;
3180
3181	const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
3182	Log3(("pgmPhysFreePage: idPage=%#x HCPhys=%RGp pPage=%R[pgmpage]\n", idPage, pPage));
3183	if (RT_UNLIKELY( idPage == NIL_GMM_PAGEID
3184	\|\| idPage > GMM_PAGEID_LAST
3185	\|\| PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID))
3186	{
3187	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
3188	return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
3189	}
3190
3191	/* update page count stats. */
3192	if (PGM_PAGE_IS_SHARED(pPage))
3193	pVM->pgm.s.cSharedPages--;
3194	else
3195	pVM->pgm.s.cPrivatePages--;
3196	pVM->pgm.s.cZeroPages++;
3197
3198	/*
3199	* pPage = ZERO page.
3200	*/
3201	PGM_PAGE_SET_HCPHYS(pPage, pVM->pgm.s.HCPhysZeroPg);
3202	PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
3203	PGM_PAGE_SET_PAGEID(pPage, NIL_GMM_PAGEID);
3204
3205	/*
3206	* Make sure it's not in the handy page array.
3207	*/
3208	for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
3209	{
3210	if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
3211	{
3212	pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
3213	break;
3214	}
3215	if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
3216	{
3217	pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
3218	break;
3219	}
3220	}
3221
3222	/*
3223	* Push it onto the page array.
3224	*/
3225	uint32_t iPage = *pcPendingPages;
3226	Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
3227	*pcPendingPages += 1;
3228
3229	pReq->aPages[iPage].idPage = idPage;
3230
3231	if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
3232	return VINF_SUCCESS;
3233
3234	/*
3235	* Flush the pages.
3236	*/
3237	int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
3238	if (RT_SUCCESS(rc))
3239	{
3240	GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
3241	*pcPendingPages = 0;
3242	}
3243	return rc;
3244	}
3245
3246
3247	/**
3248	* Converts a GC physical address to a HC ring-3 pointer, with some
3249	* additional checks.
3250	*
3251	* @returns VBox status code.
3252	* @retval VINF_SUCCESS on success.
3253	* @retval VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
3254	* access handler of some kind.
3255	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
3256	* accesses or is odd in any way.
3257	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
3258	*
3259	* @param pVM The VM handle.
3260	* @param GCPhys The GC physical address to convert.
3261	* @param fWritable Whether write access is required.
3262	* @param ppv Where to store the pointer corresponding to GCPhys on
3263	* success.
3264	*/
3265	VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
3266	{
3267	pgmLock(pVM);
3268
3269	PPGMRAMRANGE pRam;
3270	PPGMPAGE pPage;
3271	int rc = pgmPhysGetPageAndRangeEx(&pVM->pgm.s, GCPhys, &pPage, &pRam);
3272	if (RT_SUCCESS(rc))
3273	{
3274	if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
3275	rc = VINF_SUCCESS;
3276	else
3277	{
3278	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
3279	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
3280	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
3281	{
3282	/** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
3283	* in -norawr0 mode. */
3284	if (fWritable)
3285	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
3286	}
3287	else
3288	{
3289	/* Temporarily disabled physical handler(s), since the recompiler
3290	doesn't get notified when it's reset we'll have to pretend it's
3291	operating normally. */
3292	if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
3293	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
3294	else
3295	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
3296	}
3297	}
3298	if (RT_SUCCESS(rc))
3299	{
3300	int rc2;
3301
3302	/* Make sure what we return is writable. */
3303	if (fWritable && rc != VINF_PGM_PHYS_TLB_CATCH_WRITE)
3304	switch (PGM_PAGE_GET_STATE(pPage))
3305	{
3306	case PGM_PAGE_STATE_ALLOCATED:
3307	break;
3308	case PGM_PAGE_STATE_ZERO:
3309	case PGM_PAGE_STATE_SHARED:
3310	case PGM_PAGE_STATE_WRITE_MONITORED:
3311	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
3312	AssertLogRelRCReturn(rc2, rc2);
3313	break;
3314	}
3315
3316	/* Get a ring-3 mapping of the address. */
3317	PPGMPAGER3MAPTLBE pTlbe;
3318	rc2 = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
3319	AssertLogRelRCReturn(rc2, rc2);
3320	ppv = (void )((uintptr_t)pTlbe->pv \| (GCPhys & PAGE_OFFSET_MASK));
3321	/** @todo mapping/locking hell; this isn't horribly efficient since
3322	* pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
3323
3324	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
3325	}
3326	else
3327	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
3328
3329	/* else: handler catching all access, no pointer returned. */
3330	}
3331	else
3332	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
3333
3334	pgmUnlock(pVM);
3335	return rc;
3336	}
3337

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/PGMPhys.cpp@ 23488

Download in other formats: