VirtualBox

source: vbox/trunk/src/VBox/VMM/PGMPhys.cpp@ 31596

Last change on this file since 31596 was 31398, checked in by vboxsync, 14 years ago

PGMPhys.cpp: too long lines.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 143.5 KB
Line 
1/* $Id: PGMPhys.cpp 31398 2010-08-05 12:18:36Z vboxsync $ */
2/** @file
3 * PGM - Page Manager and Monitor, Physical Memory Addressing.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_PGM_PHYS
23#include <VBox/pgm.h>
24#include <VBox/iom.h>
25#include <VBox/mm.h>
26#include <VBox/stam.h>
27#include <VBox/rem.h>
28#include <VBox/pdmdev.h>
29#include "PGMInternal.h"
30#include <VBox/vm.h>
31#include "PGMInline.h"
32#include <VBox/sup.h>
33#include <VBox/param.h>
34#include <VBox/err.h>
35#include <VBox/log.h>
36#include <iprt/assert.h>
37#include <iprt/alloc.h>
38#include <iprt/asm.h>
39#include <iprt/thread.h>
40#include <iprt/string.h>
41
42
43/*******************************************************************************
44* Defined Constants And Macros *
45*******************************************************************************/
46/** The number of pages to free in one batch. */
47#define PGMPHYS_FREE_PAGE_BATCH_SIZE 128
48
49
50/*******************************************************************************
51* Internal Functions *
52*******************************************************************************/
53static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
54
55
56/*
57 * PGMR3PhysReadU8-64
58 * PGMR3PhysWriteU8-64
59 */
60#define PGMPHYSFN_READNAME PGMR3PhysReadU8
61#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
62#define PGMPHYS_DATASIZE 1
63#define PGMPHYS_DATATYPE uint8_t
64#include "PGMPhysRWTmpl.h"
65
66#define PGMPHYSFN_READNAME PGMR3PhysReadU16
67#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
68#define PGMPHYS_DATASIZE 2
69#define PGMPHYS_DATATYPE uint16_t
70#include "PGMPhysRWTmpl.h"
71
72#define PGMPHYSFN_READNAME PGMR3PhysReadU32
73#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
74#define PGMPHYS_DATASIZE 4
75#define PGMPHYS_DATATYPE uint32_t
76#include "PGMPhysRWTmpl.h"
77
78#define PGMPHYSFN_READNAME PGMR3PhysReadU64
79#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
80#define PGMPHYS_DATASIZE 8
81#define PGMPHYS_DATATYPE uint64_t
82#include "PGMPhysRWTmpl.h"
83
84
85/**
86 * EMT worker for PGMR3PhysReadExternal.
87 */
88static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead)
89{
90 PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead);
91 return VINF_SUCCESS;
92}
93
94
95/**
96 * Write to physical memory, external users.
97 *
98 * @returns VBox status code.
99 * @retval VINF_SUCCESS.
100 *
101 * @param pVM VM Handle.
102 * @param GCPhys Physical address to write to.
103 * @param pvBuf What to write.
104 * @param cbWrite How many bytes to write.
105 *
106 * @thread Any but EMTs.
107 */
108VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
109{
110 VM_ASSERT_OTHER_THREAD(pVM);
111
112 AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
113 LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
114
115 pgmLock(pVM);
116
117 /*
118 * Copy loop on ram ranges.
119 */
120 PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
121 for (;;)
122 {
123 /* Find range. */
124 while (pRam && GCPhys > pRam->GCPhysLast)
125 pRam = pRam->CTX_SUFF(pNext);
126 /* Inside range or not? */
127 if (pRam && GCPhys >= pRam->GCPhys)
128 {
129 /*
130 * Must work our way thru this page by page.
131 */
132 RTGCPHYS off = GCPhys - pRam->GCPhys;
133 while (off < pRam->cb)
134 {
135 unsigned iPage = off >> PAGE_SHIFT;
136 PPGMPAGE pPage = &pRam->aPages[iPage];
137
138 /*
139 * If the page has an ALL access handler, we'll have to
140 * delegate the job to EMT.
141 */
142 if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
143 {
144 pgmUnlock(pVM);
145
146 return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 4,
147 pVM, &GCPhys, pvBuf, cbRead);
148 }
149 Assert(!PGM_PAGE_IS_MMIO(pPage));
150
151 /*
152 * Simple stuff, go ahead.
153 */
154 size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
155 if (cb > cbRead)
156 cb = cbRead;
157 const void *pvSrc;
158 int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc);
159 if (RT_SUCCESS(rc))
160 memcpy(pvBuf, pvSrc, cb);
161 else
162 {
163 AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
164 pRam->GCPhys + off, pPage, rc));
165 memset(pvBuf, 0xff, cb);
166 }
167
168 /* next page */
169 if (cb >= cbRead)
170 {
171 pgmUnlock(pVM);
172 return VINF_SUCCESS;
173 }
174 cbRead -= cb;
175 off += cb;
176 GCPhys += cb;
177 pvBuf = (char *)pvBuf + cb;
178 } /* walk pages in ram range. */
179 }
180 else
181 {
182 LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
183
184 /*
185 * Unassigned address space.
186 */
187 if (!pRam)
188 break;
189 size_t cb = pRam->GCPhys - GCPhys;
190 if (cb >= cbRead)
191 {
192 memset(pvBuf, 0xff, cbRead);
193 break;
194 }
195 memset(pvBuf, 0xff, cb);
196
197 cbRead -= cb;
198 pvBuf = (char *)pvBuf + cb;
199 GCPhys += cb;
200 }
201 } /* Ram range walk */
202
203 pgmUnlock(pVM);
204
205 return VINF_SUCCESS;
206}
207
208
209/**
210 * EMT worker for PGMR3PhysWriteExternal.
211 */
212static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite)
213{
214 /** @todo VERR_EM_NO_MEMORY */
215 PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite);
216 return VINF_SUCCESS;
217}
218
219
220/**
221 * Write to physical memory, external users.
222 *
223 * @returns VBox status code.
224 * @retval VINF_SUCCESS.
225 * @retval VERR_EM_NO_MEMORY.
226 *
227 * @param pVM VM Handle.
228 * @param GCPhys Physical address to write to.
229 * @param pvBuf What to write.
230 * @param cbWrite How many bytes to write.
231 * @param pszWho Who is writing. For tracking down who is writing
232 * after we've saved the state.
233 *
234 * @thread Any but EMTs.
235 */
236VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, const char *pszWho)
237{
238 VM_ASSERT_OTHER_THREAD(pVM);
239
240 AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
241 ("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x pszWho=%s\n",
242 GCPhys, cbWrite, pszWho));
243 AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
244 LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
245
246 pgmLock(pVM);
247
248 /*
249 * Copy loop on ram ranges, stop when we hit something difficult.
250 */
251 PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
252 for (;;)
253 {
254 /* Find range. */
255 while (pRam && GCPhys > pRam->GCPhysLast)
256 pRam = pRam->CTX_SUFF(pNext);
257 /* Inside range or not? */
258 if (pRam && GCPhys >= pRam->GCPhys)
259 {
260 /*
261 * Must work our way thru this page by page.
262 */
263 RTGCPTR off = GCPhys - pRam->GCPhys;
264 while (off < pRam->cb)
265 {
266 RTGCPTR iPage = off >> PAGE_SHIFT;
267 PPGMPAGE pPage = &pRam->aPages[iPage];
268
269 /*
270 * Is the page problematic, we have to do the work on the EMT.
271 *
272 * Allocating writable pages and access handlers are
273 * problematic, write monitored pages are simple and can be
274 * dealth with here.
275 */
276 if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
277 || PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
278 {
279 if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
280 && !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
281 pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
282 else
283 {
284 pgmUnlock(pVM);
285
286 return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 4,
287 pVM, &GCPhys, pvBuf, cbWrite);
288 }
289 }
290 Assert(!PGM_PAGE_IS_MMIO(pPage));
291
292 /*
293 * Simple stuff, go ahead.
294 */
295 size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
296 if (cb > cbWrite)
297 cb = cbWrite;
298 void *pvDst;
299 int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst);
300 if (RT_SUCCESS(rc))
301 memcpy(pvDst, pvBuf, cb);
302 else
303 AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
304 pRam->GCPhys + off, pPage, rc));
305
306 /* next page */
307 if (cb >= cbWrite)
308 {
309 pgmUnlock(pVM);
310 return VINF_SUCCESS;
311 }
312
313 cbWrite -= cb;
314 off += cb;
315 GCPhys += cb;
316 pvBuf = (const char *)pvBuf + cb;
317 } /* walk pages in ram range */
318 }
319 else
320 {
321 /*
322 * Unassigned address space, skip it.
323 */
324 if (!pRam)
325 break;
326 size_t cb = pRam->GCPhys - GCPhys;
327 if (cb >= cbWrite)
328 break;
329 cbWrite -= cb;
330 pvBuf = (const char *)pvBuf + cb;
331 GCPhys += cb;
332 }
333 } /* Ram range walk */
334
335 pgmUnlock(pVM);
336 return VINF_SUCCESS;
337}
338
339
340/**
341 * VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
342 *
343 * @returns see PGMR3PhysGCPhys2CCPtrExternal
344 * @param pVM The VM handle.
345 * @param pGCPhys Pointer to the guest physical address.
346 * @param ppv Where to store the mapping address.
347 * @param pLock Where to store the lock.
348 */
349static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
350{
351 /*
352 * Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
353 * an access handler after it succeeds.
354 */
355 int rc = pgmLock(pVM);
356 AssertRCReturn(rc, rc);
357
358 rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
359 if (RT_SUCCESS(rc))
360 {
361 PPGMPAGEMAPTLBE pTlbe;
362 int rc2 = pgmPhysPageQueryTlbe(&pVM->pgm.s, *pGCPhys, &pTlbe);
363 AssertFatalRC(rc2);
364 PPGMPAGE pPage = pTlbe->pPage;
365 if (PGM_PAGE_IS_MMIO(pPage))
366 {
367 PGMPhysReleasePageMappingLock(pVM, pLock);
368 rc = VERR_PGM_PHYS_PAGE_RESERVED;
369 }
370 else if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
371#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
372 || pgmPoolIsDirtyPage(pVM, *pGCPhys)
373#endif
374 )
375 {
376 /* We *must* flush any corresponding pgm pool page here, otherwise we'll
377 * not be informed about writes and keep bogus gst->shw mappings around.
378 */
379 pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
380 Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
381 /** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
382 * active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
383 }
384 }
385
386 pgmUnlock(pVM);
387 return rc;
388}
389
390
391/**
392 * Requests the mapping of a guest page into ring-3, external threads.
393 *
394 * When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
395 * release it.
396 *
397 * This API will assume your intention is to write to the page, and will
398 * therefore replace shared and zero pages. If you do not intend to modify the
399 * page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
400 *
401 * @returns VBox status code.
402 * @retval VINF_SUCCESS on success.
403 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
404 * backing or if the page has any active access handlers. The caller
405 * must fall back on using PGMR3PhysWriteExternal.
406 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
407 *
408 * @param pVM The VM handle.
409 * @param GCPhys The guest physical address of the page that should be mapped.
410 * @param ppv Where to store the address corresponding to GCPhys.
411 * @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
412 *
413 * @remark Avoid calling this API from within critical sections (other than the
414 * PGM one) because of the deadlock risk when we have to delegating the
415 * task to an EMT.
416 * @thread Any.
417 */
418VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
419{
420 AssertPtr(ppv);
421 AssertPtr(pLock);
422
423 Assert(VM_IS_EMT(pVM) || !PGMIsLockOwner(pVM));
424
425 int rc = pgmLock(pVM);
426 AssertRCReturn(rc, rc);
427
428 /*
429 * Query the Physical TLB entry for the page (may fail).
430 */
431 PPGMPAGEMAPTLBE pTlbe;
432 rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
433 if (RT_SUCCESS(rc))
434 {
435 PPGMPAGE pPage = pTlbe->pPage;
436 if (PGM_PAGE_IS_MMIO(pPage))
437 rc = VERR_PGM_PHYS_PAGE_RESERVED;
438 else
439 {
440 /*
441 * If the page is shared, the zero page, or being write monitored
442 * it must be converted to an page that's writable if possible.
443 * We can only deal with write monitored pages here, the rest have
444 * to be on an EMT.
445 */
446 if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
447 || PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
448#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
449 || pgmPoolIsDirtyPage(pVM, GCPhys)
450#endif
451 )
452 {
453 if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
454 && !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
455#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
456 && !pgmPoolIsDirtyPage(pVM, GCPhys)
457#endif
458 )
459 pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
460 else
461 {
462 pgmUnlock(pVM);
463
464 return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
465 pVM, &GCPhys, ppv, pLock);
466 }
467 }
468
469 /*
470 * Now, just perform the locking and calculate the return address.
471 */
472 PPGMPAGEMAP pMap = pTlbe->pMap;
473 if (pMap)
474 pMap->cRefs++;
475
476 unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
477 if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
478 {
479 if (cLocks == 0)
480 pVM->pgm.s.cWriteLockedPages++;
481 PGM_PAGE_INC_WRITE_LOCKS(pPage);
482 }
483 else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
484 {
485 PGM_PAGE_INC_WRITE_LOCKS(pPage);
486 AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
487 if (pMap)
488 pMap->cRefs++; /* Extra ref to prevent it from going away. */
489 }
490
491 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
492 pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_WRITE;
493 pLock->pvMap = pMap;
494 }
495 }
496
497 pgmUnlock(pVM);
498 return rc;
499}
500
501
502/**
503 * Requests the mapping of a guest page into ring-3, external threads.
504 *
505 * When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
506 * release it.
507 *
508 * @returns VBox status code.
509 * @retval VINF_SUCCESS on success.
510 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
511 * backing or if the page as an active ALL access handler. The caller
512 * must fall back on using PGMPhysRead.
513 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
514 *
515 * @param pVM The VM handle.
516 * @param GCPhys The guest physical address of the page that should be mapped.
517 * @param ppv Where to store the address corresponding to GCPhys.
518 * @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
519 *
520 * @remark Avoid calling this API from within critical sections (other than
521 * the PGM one) because of the deadlock risk.
522 * @thread Any.
523 */
524VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
525{
526 int rc = pgmLock(pVM);
527 AssertRCReturn(rc, rc);
528
529 /*
530 * Query the Physical TLB entry for the page (may fail).
531 */
532 PPGMPAGEMAPTLBE pTlbe;
533 rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
534 if (RT_SUCCESS(rc))
535 {
536 PPGMPAGE pPage = pTlbe->pPage;
537#if 1
538 /* MMIO pages doesn't have any readable backing. */
539 if (PGM_PAGE_IS_MMIO(pPage))
540 rc = VERR_PGM_PHYS_PAGE_RESERVED;
541#else
542 if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
543 rc = VERR_PGM_PHYS_PAGE_RESERVED;
544#endif
545 else
546 {
547 /*
548 * Now, just perform the locking and calculate the return address.
549 */
550 PPGMPAGEMAP pMap = pTlbe->pMap;
551 if (pMap)
552 pMap->cRefs++;
553
554 unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
555 if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
556 {
557 if (cLocks == 0)
558 pVM->pgm.s.cReadLockedPages++;
559 PGM_PAGE_INC_READ_LOCKS(pPage);
560 }
561 else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
562 {
563 PGM_PAGE_INC_READ_LOCKS(pPage);
564 AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
565 if (pMap)
566 pMap->cRefs++; /* Extra ref to prevent it from going away. */
567 }
568
569 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
570 pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_READ;
571 pLock->pvMap = pMap;
572 }
573 }
574
575 pgmUnlock(pVM);
576 return rc;
577}
578
579
580/**
581 * Relinks the RAM ranges using the pSelfRC and pSelfR0 pointers.
582 *
583 * Called when anything was relocated.
584 *
585 * @param pVM Pointer to the shared VM structure.
586 */
587void pgmR3PhysRelinkRamRanges(PVM pVM)
588{
589 PPGMRAMRANGE pCur;
590
591#ifdef VBOX_STRICT
592 for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
593 {
594 Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pCur->pSelfR0 == MMHyperCCToR0(pVM, pCur));
595 Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pCur->pSelfRC == MMHyperCCToRC(pVM, pCur));
596 Assert((pCur->GCPhys & PAGE_OFFSET_MASK) == 0);
597 Assert((pCur->GCPhysLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
598 Assert((pCur->cb & PAGE_OFFSET_MASK) == 0);
599 Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
600 for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesR3; pCur2; pCur2 = pCur2->pNextR3)
601 Assert( pCur2 == pCur
602 || strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
603 }
604#endif
605
606 pCur = pVM->pgm.s.pRamRangesR3;
607 if (pCur)
608 {
609 pVM->pgm.s.pRamRangesR0 = pCur->pSelfR0;
610 pVM->pgm.s.pRamRangesRC = pCur->pSelfRC;
611
612 for (; pCur->pNextR3; pCur = pCur->pNextR3)
613 {
614 pCur->pNextR0 = pCur->pNextR3->pSelfR0;
615 pCur->pNextRC = pCur->pNextR3->pSelfRC;
616 }
617
618 Assert(pCur->pNextR0 == NIL_RTR0PTR);
619 Assert(pCur->pNextRC == NIL_RTRCPTR);
620 }
621 else
622 {
623 Assert(pVM->pgm.s.pRamRangesR0 == NIL_RTR0PTR);
624 Assert(pVM->pgm.s.pRamRangesRC == NIL_RTRCPTR);
625 }
626 ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
627}
628
629
630/**
631 * Links a new RAM range into the list.
632 *
633 * @param pVM Pointer to the shared VM structure.
634 * @param pNew Pointer to the new list entry.
635 * @param pPrev Pointer to the previous list entry. If NULL, insert as head.
636 */
637static void pgmR3PhysLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, PPGMRAMRANGE pPrev)
638{
639 AssertMsg(pNew->pszDesc, ("%RGp-%RGp\n", pNew->GCPhys, pNew->GCPhysLast));
640 Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pNew->pSelfR0 == MMHyperCCToR0(pVM, pNew));
641 Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pNew->pSelfRC == MMHyperCCToRC(pVM, pNew));
642
643 pgmLock(pVM);
644
645 PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesR3;
646 pNew->pNextR3 = pRam;
647 pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
648 pNew->pNextRC = pRam ? pRam->pSelfRC : NIL_RTRCPTR;
649
650 if (pPrev)
651 {
652 pPrev->pNextR3 = pNew;
653 pPrev->pNextR0 = pNew->pSelfR0;
654 pPrev->pNextRC = pNew->pSelfRC;
655 }
656 else
657 {
658 pVM->pgm.s.pRamRangesR3 = pNew;
659 pVM->pgm.s.pRamRangesR0 = pNew->pSelfR0;
660 pVM->pgm.s.pRamRangesRC = pNew->pSelfRC;
661 }
662 ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
663 pgmUnlock(pVM);
664}
665
666
667/**
668 * Unlink an existing RAM range from the list.
669 *
670 * @param pVM Pointer to the shared VM structure.
671 * @param pRam Pointer to the new list entry.
672 * @param pPrev Pointer to the previous list entry. If NULL, insert as head.
673 */
674static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
675{
676 Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesR3 == pRam);
677 Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pRam->pSelfR0 == MMHyperCCToR0(pVM, pRam));
678 Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pRam->pSelfRC == MMHyperCCToRC(pVM, pRam));
679
680 pgmLock(pVM);
681
682 PPGMRAMRANGE pNext = pRam->pNextR3;
683 if (pPrev)
684 {
685 pPrev->pNextR3 = pNext;
686 pPrev->pNextR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
687 pPrev->pNextRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
688 }
689 else
690 {
691 Assert(pVM->pgm.s.pRamRangesR3 == pRam);
692 pVM->pgm.s.pRamRangesR3 = pNext;
693 pVM->pgm.s.pRamRangesR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
694 pVM->pgm.s.pRamRangesRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
695 }
696 ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
697 pgmUnlock(pVM);
698}
699
700
701/**
702 * Unlink an existing RAM range from the list.
703 *
704 * @param pVM Pointer to the shared VM structure.
705 * @param pRam Pointer to the new list entry.
706 */
707static void pgmR3PhysUnlinkRamRange(PVM pVM, PPGMRAMRANGE pRam)
708{
709 pgmLock(pVM);
710
711 /* find prev. */
712 PPGMRAMRANGE pPrev = NULL;
713 PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesR3;
714 while (pCur != pRam)
715 {
716 pPrev = pCur;
717 pCur = pCur->pNextR3;
718 }
719 AssertFatal(pCur);
720
721 pgmR3PhysUnlinkRamRange2(pVM, pRam, pPrev);
722 pgmUnlock(pVM);
723}
724
725
726/**
727 * Frees a range of pages, replacing them with ZERO pages of the specified type.
728 *
729 * @returns VBox status code.
730 * @param pVM The VM handle.
731 * @param pRam The RAM range in which the pages resides.
732 * @param GCPhys The address of the first page.
733 * @param GCPhysLast The address of the last page.
734 * @param uType The page type to replace then with.
735 */
736static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, uint8_t uType)
737{
738 Assert(PGMIsLockOwner(pVM));
739 uint32_t cPendingPages = 0;
740 PGMMFREEPAGESREQ pReq;
741 int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
742 AssertLogRelRCReturn(rc, rc);
743
744 /* Iterate the pages. */
745 PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
746 uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> PAGE_SHIFT) + 1;
747 while (cPagesLeft-- > 0)
748 {
749 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
750 AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
751
752 PGM_PAGE_SET_TYPE(pPageDst, uType);
753
754 GCPhys += PAGE_SIZE;
755 pPageDst++;
756 }
757
758 if (cPendingPages)
759 {
760 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
761 AssertLogRelRCReturn(rc, rc);
762 }
763 GMMR3FreePagesCleanup(pReq);
764
765 return rc;
766}
767
768#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) || defined(RT_OS_SOLARIS) || defined(RT_OS_LINUX) || defined(RT_OS_FREEBSD))
769/**
770 * Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
771 *
772 * This is only called on one of the EMTs while the other ones are waiting for
773 * it to complete this function.
774 *
775 * @returns VINF_SUCCESS (VBox strict status code).
776 * @param pVM The VM handle.
777 * @param pVCpu The VMCPU for the EMT we're being called on. Unused.
778 * @param pvUser User parameter
779 */
780static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
781{
782 uintptr_t *paUser = (uintptr_t *)pvUser;
783 bool fInflate = !!paUser[0];
784 unsigned cPages = paUser[1];
785 RTGCPHYS *paPhysPage = (RTGCPHYS *)paUser[2];
786 uint32_t cPendingPages = 0;
787 PGMMFREEPAGESREQ pReq;
788 int rc;
789
790 Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
791 pgmLock(pVM);
792
793 if (fInflate)
794 {
795 /* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
796 pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
797
798 /* Replace pages with ZERO pages. */
799 rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
800 if (RT_FAILURE(rc))
801 {
802 pgmUnlock(pVM);
803 AssertLogRelRC(rc);
804 return rc;
805 }
806
807 /* Iterate the pages. */
808 for (unsigned i = 0; i < cPages; i++)
809 {
810 PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, paPhysPage[i]);
811 if ( pPage == NULL
812 || pPage->uTypeY != PGMPAGETYPE_RAM)
813 {
814 Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], (pPage) ? pPage->uTypeY : 0));
815 break;
816 }
817
818 LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
819
820 /* Flush the shadow PT if this page was previously used as a guest page table. */
821 pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
822
823 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i]);
824 if (RT_FAILURE(rc))
825 {
826 pgmUnlock(pVM);
827 AssertLogRelRC(rc);
828 return rc;
829 }
830 Assert(PGM_PAGE_IS_ZERO(pPage));
831 PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_BALLOONED);
832 }
833
834 if (cPendingPages)
835 {
836 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
837 if (RT_FAILURE(rc))
838 {
839 pgmUnlock(pVM);
840 AssertLogRelRC(rc);
841 return rc;
842 }
843 }
844 GMMR3FreePagesCleanup(pReq);
845 }
846 else
847 {
848 /* Iterate the pages. */
849 for (unsigned i = 0; i < cPages; i++)
850 {
851 PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, paPhysPage[i]);
852 AssertBreak(pPage && pPage->uTypeY == PGMPAGETYPE_RAM);
853
854 LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
855
856 Assert(PGM_PAGE_IS_BALLOONED(pPage));
857
858 /* Change back to zero page. */
859 PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
860 }
861
862 /* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
863 }
864
865 /* Notify GMM about the balloon change. */
866 rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
867 if (RT_SUCCESS(rc))
868 {
869 if (!fInflate)
870 {
871 Assert(pVM->pgm.s.cBalloonedPages >= cPages);
872 pVM->pgm.s.cBalloonedPages -= cPages;
873 }
874 else
875 pVM->pgm.s.cBalloonedPages += cPages;
876 }
877
878 pgmUnlock(pVM);
879
880 /* Flush the recompiler's TLB as well. */
881 for (VMCPUID i = 0; i < pVM->cCpus; i++)
882 CPUMSetChangedFlags(&pVM->aCpus[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
883
884 AssertLogRelRC(rc);
885 return rc;
886}
887
888/**
889 * Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
890 *
891 * @returns VBox status code.
892 * @param pVM The VM handle.
893 * @param fInflate Inflate or deflate memory balloon
894 * @param cPages Number of pages to free
895 * @param paPhysPage Array of guest physical addresses
896 */
897static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
898{
899 uintptr_t paUser[3];
900
901 paUser[0] = fInflate;
902 paUser[1] = cPages;
903 paUser[2] = (uintptr_t)paPhysPage;
904 int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
905 AssertRC(rc);
906
907 /* Made a copy in PGMR3PhysFreeRamPages; free it here. */
908 RTMemFree(paPhysPage);
909}
910#endif
911
912/**
913 * Inflate or deflate a memory balloon
914 *
915 * @returns VBox status code.
916 * @param pVM The VM handle.
917 * @param fInflate Inflate or deflate memory balloon
918 * @param cPages Number of pages to free
919 * @param paPhysPage Array of guest physical addresses
920 */
921VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
922{
923 /* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
924#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) || defined(RT_OS_SOLARIS) || defined(RT_OS_LINUX) || defined(RT_OS_FREEBSD))
925 int rc;
926
927 /* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
928 AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
929
930 /* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
931 * In the SMP case we post a request packet to postpone the job.
932 */
933 if (pVM->cCpus > 1)
934 {
935 unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
936 RTGCPHYS *paPhysPageCopy = (RTGCPHYS *)RTMemAlloc(cbPhysPage);
937 AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
938
939 memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
940
941 rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4, pVM, fInflate, cPages, paPhysPageCopy);
942 AssertRC(rc);
943 }
944 else
945 {
946 uintptr_t paUser[3];
947
948 paUser[0] = fInflate;
949 paUser[1] = cPages;
950 paUser[2] = (uintptr_t)paPhysPage;
951 rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
952 AssertRC(rc);
953 }
954 return rc;
955#else
956 return VERR_NOT_IMPLEMENTED;
957#endif
958}
959
960/**
961 * Query the amount of free memory inside VMMR0
962 *
963 * @returns VBox status code.
964 * @param pVM The VM handle.
965 * @param puTotalAllocSize Pointer to total allocated memory inside VMMR0 (in bytes)
966 * @param puTotalFreeSize Pointer to total free (allocated but not used yet) memory inside VMMR0 (in bytes)
967 * @param puTotalBalloonSize Pointer to total ballooned memory inside VMMR0 (in bytes)
968 * @param puTotalSharedSize Pointer to total shared memory inside VMMR0 (in bytes)
969 */
970VMMR3DECL(int) PGMR3QueryVMMMemoryStats(PVM pVM, uint64_t *puTotalAllocSize, uint64_t *puTotalFreeSize, uint64_t *puTotalBalloonSize, uint64_t *puTotalSharedSize)
971{
972 int rc;
973
974 uint64_t cAllocPages = 0, cFreePages = 0, cBalloonPages = 0, cSharedPages = 0;
975 rc = GMMR3QueryHypervisorMemoryStats(pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
976 AssertRCReturn(rc, rc);
977
978 if (puTotalAllocSize)
979 *puTotalAllocSize = cAllocPages * _4K;
980
981 if (puTotalFreeSize)
982 *puTotalFreeSize = cFreePages * _4K;
983
984 if (puTotalBalloonSize)
985 *puTotalBalloonSize = cBalloonPages * _4K;
986
987 if (puTotalSharedSize)
988 *puTotalSharedSize = cSharedPages * _4K;
989
990 Log(("PGMR3QueryVMMMemoryStats: all=%x free=%x ballooned=%x shared=%x\n", cAllocPages, cFreePages, cBalloonPages, cSharedPages));
991 return VINF_SUCCESS;
992}
993
994/**
995 * Query memory stats for the VM
996 *
997 * @returns VBox status code.
998 * @param pVM The VM handle.
999 * @param puTotalAllocSize Pointer to total allocated memory inside the VM (in bytes)
1000 * @param puTotalFreeSize Pointer to total free (allocated but not used yet) memory inside the VM (in bytes)
1001 * @param puTotalBalloonSize Pointer to total ballooned memory inside the VM (in bytes)
1002 * @param puTotalSharedSize Pointer to total shared memory inside the VM (in bytes)
1003 */
1004VMMR3DECL(int) PGMR3QueryMemoryStats(PVM pVM, uint64_t *pulTotalMem, uint64_t *pulPrivateMem, uint64_t *puTotalSharedMem, uint64_t *puTotalZeroMem)
1005{
1006 if (pulTotalMem)
1007 *pulTotalMem = (uint64_t)pVM->pgm.s.cAllPages * _4K;
1008
1009 if (pulPrivateMem)
1010 *pulPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages * _4K;
1011
1012 if (puTotalSharedMem)
1013 *puTotalSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages * _4K;
1014
1015 if (puTotalZeroMem)
1016 *puTotalZeroMem = (uint64_t)pVM->pgm.s.cZeroPages * _4K;
1017
1018 Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
1019 return VINF_SUCCESS;
1020}
1021
1022/**
1023 * PGMR3PhysRegisterRam worker that initializes and links a RAM range.
1024 *
1025 * @param pVM The VM handle.
1026 * @param pNew The new RAM range.
1027 * @param GCPhys The address of the RAM range.
1028 * @param GCPhysLast The last address of the RAM range.
1029 * @param RCPtrNew The RC address if the range is floating. NIL_RTRCPTR
1030 * if in HMA.
1031 * @param R0PtrNew Ditto for R0.
1032 * @param pszDesc The description.
1033 * @param pPrev The previous RAM range (for linking).
1034 */
1035static void pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
1036 RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
1037{
1038 /*
1039 * Initialize the range.
1040 */
1041 pNew->pSelfR0 = R0PtrNew != NIL_RTR0PTR ? R0PtrNew : MMHyperCCToR0(pVM, pNew);
1042 pNew->pSelfRC = RCPtrNew != NIL_RTRCPTR ? RCPtrNew : MMHyperCCToRC(pVM, pNew);
1043 pNew->GCPhys = GCPhys;
1044 pNew->GCPhysLast = GCPhysLast;
1045 pNew->cb = GCPhysLast - GCPhys + 1;
1046 pNew->pszDesc = pszDesc;
1047 pNew->fFlags = RCPtrNew != NIL_RTRCPTR ? PGM_RAM_RANGE_FLAGS_FLOATING : 0;
1048 pNew->pvR3 = NULL;
1049 pNew->paLSPages = NULL;
1050
1051 uint32_t const cPages = pNew->cb >> PAGE_SHIFT;
1052 RTGCPHYS iPage = cPages;
1053 while (iPage-- > 0)
1054 PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
1055
1056 /* Update the page count stats. */
1057 pVM->pgm.s.cZeroPages += cPages;
1058 pVM->pgm.s.cAllPages += cPages;
1059
1060 /*
1061 * Link it.
1062 */
1063 pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
1064}
1065
1066
1067/**
1068 * Relocate a floating RAM range.
1069 *
1070 * @copydoc FNPGMRELOCATE.
1071 */
1072static DECLCALLBACK(bool) pgmR3PhysRamRangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser)
1073{
1074 PPGMRAMRANGE pRam = (PPGMRAMRANGE)pvUser;
1075 Assert(pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
1076 Assert(pRam->pSelfRC == GCPtrOld + PAGE_SIZE);
1077
1078 switch (enmMode)
1079 {
1080 case PGMRELOCATECALL_SUGGEST:
1081 return true;
1082 case PGMRELOCATECALL_RELOCATE:
1083 {
1084 /* Update myself and then relink all the ranges. */
1085 pgmLock(pVM);
1086 pRam->pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE);
1087 pgmR3PhysRelinkRamRanges(pVM);
1088 pgmUnlock(pVM);
1089 return true;
1090 }
1091
1092 default:
1093 AssertFailedReturn(false);
1094 }
1095}
1096
1097
1098/**
1099 * PGMR3PhysRegisterRam worker that registers a high chunk.
1100 *
1101 * @returns VBox status code.
1102 * @param pVM The VM handle.
1103 * @param GCPhys The address of the RAM.
1104 * @param cRamPages The number of RAM pages to register.
1105 * @param cbChunk The size of the PGMRAMRANGE guest mapping.
1106 * @param iChunk The chunk number.
1107 * @param pszDesc The RAM range description.
1108 * @param ppPrev Previous RAM range pointer. In/Out.
1109 */
1110static int pgmR3PhysRegisterHighRamChunk(PVM pVM, RTGCPHYS GCPhys, uint32_t cRamPages,
1111 uint32_t cbChunk, uint32_t iChunk, const char *pszDesc,
1112 PPGMRAMRANGE *ppPrev)
1113{
1114 const char *pszDescChunk = iChunk == 0
1115 ? pszDesc
1116 : MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, iChunk + 1);
1117 AssertReturn(pszDescChunk, VERR_NO_MEMORY);
1118
1119 /*
1120 * Allocate memory for the new chunk.
1121 */
1122 size_t const cChunkPages = RT_ALIGN_Z(RT_UOFFSETOF(PGMRAMRANGE, aPages[cRamPages]), PAGE_SIZE) >> PAGE_SHIFT;
1123 PSUPPAGE paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
1124 AssertReturn(paChunkPages, VERR_NO_TMP_MEMORY);
1125 RTR0PTR R0PtrChunk = NIL_RTR0PTR;
1126 void *pvChunk = NULL;
1127 int rc = SUPR3PageAllocEx(cChunkPages, 0 /*fFlags*/, &pvChunk,
1128#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1129 VMMIsHwVirtExtForced(pVM) ? &R0PtrChunk : NULL,
1130#else
1131 NULL,
1132#endif
1133 paChunkPages);
1134 if (RT_SUCCESS(rc))
1135 {
1136#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1137 if (!VMMIsHwVirtExtForced(pVM))
1138 R0PtrChunk = NIL_RTR0PTR;
1139#else
1140 R0PtrChunk = (uintptr_t)pvChunk;
1141#endif
1142 memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
1143
1144 PPGMRAMRANGE pNew = (PPGMRAMRANGE)pvChunk;
1145
1146 /*
1147 * Create a mapping and map the pages into it.
1148 * We push these in below the HMA.
1149 */
1150 RTGCPTR GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
1151 rc = PGMR3MapPT(pVM, GCPtrChunkMap, cbChunk, 0 /*fFlags*/, pgmR3PhysRamRangeRelocate, pNew, pszDescChunk);
1152 if (RT_SUCCESS(rc))
1153 {
1154 pVM->pgm.s.GCPtrPrevRamRangeMapping = GCPtrChunkMap;
1155
1156 RTGCPTR const GCPtrChunk = GCPtrChunkMap + PAGE_SIZE;
1157 RTGCPTR GCPtrPage = GCPtrChunk;
1158 for (uint32_t iPage = 0; iPage < cChunkPages && RT_SUCCESS(rc); iPage++, GCPtrPage += PAGE_SIZE)
1159 rc = PGMMap(pVM, GCPtrPage, paChunkPages[iPage].Phys, PAGE_SIZE, 0);
1160 if (RT_SUCCESS(rc))
1161 {
1162 /*
1163 * Ok, init and link the range.
1164 */
1165 pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
1166 (RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
1167 *ppPrev = pNew;
1168 }
1169 }
1170
1171 if (RT_FAILURE(rc))
1172 SUPR3PageFreeEx(pvChunk, cChunkPages);
1173 }
1174
1175 RTMemTmpFree(paChunkPages);
1176 return rc;
1177}
1178
1179
1180/**
1181 * Sets up a range RAM.
1182 *
1183 * This will check for conflicting registrations, make a resource
1184 * reservation for the memory (with GMM), and setup the per-page
1185 * tracking structures (PGMPAGE).
1186 *
1187 * @returns VBox stutus code.
1188 * @param pVM Pointer to the shared VM structure.
1189 * @param GCPhys The physical address of the RAM.
1190 * @param cb The size of the RAM.
1191 * @param pszDesc The description - not copied, so, don't free or change it.
1192 */
1193VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
1194{
1195 /*
1196 * Validate input.
1197 */
1198 Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
1199 AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
1200 AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
1201 AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
1202 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1203 AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
1204 AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1205 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1206
1207 pgmLock(pVM);
1208
1209 /*
1210 * Find range location and check for conflicts.
1211 * (We don't lock here because the locking by EMT is only required on update.)
1212 */
1213 PPGMRAMRANGE pPrev = NULL;
1214 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
1215 while (pRam && GCPhysLast >= pRam->GCPhys)
1216 {
1217 if ( GCPhysLast >= pRam->GCPhys
1218 && GCPhys <= pRam->GCPhysLast)
1219 AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
1220 GCPhys, GCPhysLast, pszDesc,
1221 pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
1222 VERR_PGM_RAM_CONFLICT);
1223
1224 /* next */
1225 pPrev = pRam;
1226 pRam = pRam->pNextR3;
1227 }
1228
1229 /*
1230 * Register it with GMM (the API bitches).
1231 */
1232 const RTGCPHYS cPages = cb >> PAGE_SHIFT;
1233 int rc = MMR3IncreaseBaseReservation(pVM, cPages);
1234 if (RT_FAILURE(rc))
1235 {
1236 pgmUnlock(pVM);
1237 return rc;
1238 }
1239
1240 if ( GCPhys >= _4G
1241 && cPages > 256)
1242 {
1243 /*
1244 * The PGMRAMRANGE structures for the high memory can get very big.
1245 * In order to avoid SUPR3PageAllocEx allocation failures due to the
1246 * allocation size limit there and also to avoid being unable to find
1247 * guest mapping space for them, we split this memory up into 4MB in
1248 * (potential) raw-mode configs and 16MB chunks in forced AMD-V/VT-x
1249 * mode.
1250 *
1251 * The first and last page of each mapping are guard pages and marked
1252 * not-present. So, we've got 4186112 and 16769024 bytes available for
1253 * the PGMRAMRANGE structure.
1254 *
1255 * Note! The sizes used here will influence the saved state.
1256 */
1257 uint32_t cbChunk;
1258 uint32_t cPagesPerChunk;
1259 if (VMMIsHwVirtExtForced(pVM))
1260 {
1261 cbChunk = 16U*_1M;
1262 cPagesPerChunk = 1048048; /* max ~1048059 */
1263 AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 1048048 < 16U*_1M - PAGE_SIZE * 2);
1264 }
1265 else
1266 {
1267 cbChunk = 4U*_1M;
1268 cPagesPerChunk = 261616; /* max ~261627 */
1269 AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 261616 < 4U*_1M - PAGE_SIZE * 2);
1270 }
1271 AssertRelease(RT_UOFFSETOF(PGMRAMRANGE, aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
1272
1273 RTGCPHYS cPagesLeft = cPages;
1274 RTGCPHYS GCPhysChunk = GCPhys;
1275 uint32_t iChunk = 0;
1276 while (cPagesLeft > 0)
1277 {
1278 uint32_t cPagesInChunk = cPagesLeft;
1279 if (cPagesInChunk > cPagesPerChunk)
1280 cPagesInChunk = cPagesPerChunk;
1281
1282 rc = pgmR3PhysRegisterHighRamChunk(pVM, GCPhysChunk, cPagesInChunk, cbChunk, iChunk, pszDesc, &pPrev);
1283 AssertRCReturn(rc, rc);
1284
1285 /* advance */
1286 GCPhysChunk += (RTGCPHYS)cPagesInChunk << PAGE_SHIFT;
1287 cPagesLeft -= cPagesInChunk;
1288 iChunk++;
1289 }
1290 }
1291 else
1292 {
1293 /*
1294 * Allocate, initialize and link the new RAM range.
1295 */
1296 const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
1297 PPGMRAMRANGE pNew;
1298 rc = MMR3HyperAllocOnceNoRel(pVM, cbRamRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
1299 AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
1300
1301 pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, NIL_RTR0PTR, pszDesc, pPrev);
1302 }
1303 PGMPhysInvalidatePageMapTLB(pVM);
1304 pgmUnlock(pVM);
1305
1306 /*
1307 * Notify REM.
1308 */
1309 REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_RAM);
1310
1311 return VINF_SUCCESS;
1312}
1313
1314
1315/**
1316 * Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
1317 *
1318 * We do this late in the init process so that all the ROM and MMIO ranges have
1319 * been registered already and we don't go wasting memory on them.
1320 *
1321 * @returns VBox status code.
1322 *
1323 * @param pVM Pointer to the shared VM structure.
1324 */
1325int pgmR3PhysRamPreAllocate(PVM pVM)
1326{
1327 Assert(pVM->pgm.s.fRamPreAlloc);
1328 Log(("pgmR3PhysRamPreAllocate: enter\n"));
1329
1330 /*
1331 * Walk the RAM ranges and allocate all RAM pages, halt at
1332 * the first allocation error.
1333 */
1334 uint64_t cPages = 0;
1335 uint64_t NanoTS = RTTimeNanoTS();
1336 pgmLock(pVM);
1337 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
1338 {
1339 PPGMPAGE pPage = &pRam->aPages[0];
1340 RTGCPHYS GCPhys = pRam->GCPhys;
1341 uint32_t cLeft = pRam->cb >> PAGE_SHIFT;
1342 while (cLeft-- > 0)
1343 {
1344 if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1345 {
1346 switch (PGM_PAGE_GET_STATE(pPage))
1347 {
1348 case PGM_PAGE_STATE_ZERO:
1349 {
1350 int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
1351 if (RT_FAILURE(rc))
1352 {
1353 LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
1354 pgmUnlock(pVM);
1355 return rc;
1356 }
1357 cPages++;
1358 break;
1359 }
1360
1361 case PGM_PAGE_STATE_BALLOONED:
1362 case PGM_PAGE_STATE_ALLOCATED:
1363 case PGM_PAGE_STATE_WRITE_MONITORED:
1364 case PGM_PAGE_STATE_SHARED:
1365 /* nothing to do here. */
1366 break;
1367 }
1368 }
1369
1370 /* next */
1371 pPage++;
1372 GCPhys += PAGE_SIZE;
1373 }
1374 }
1375 pgmUnlock(pVM);
1376 NanoTS = RTTimeNanoTS() - NanoTS;
1377
1378 LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
1379 Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
1380 return VINF_SUCCESS;
1381}
1382
1383
1384/**
1385 * Resets (zeros) the RAM.
1386 *
1387 * ASSUMES that the caller owns the PGM lock.
1388 *
1389 * @returns VBox status code.
1390 * @param pVM Pointer to the shared VM structure.
1391 */
1392int pgmR3PhysRamReset(PVM pVM)
1393{
1394 Assert(PGMIsLockOwner(pVM));
1395
1396 /* Reset the memory balloon. */
1397 int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1398 AssertRC(rc);
1399
1400#ifdef VBOX_WITH_PAGE_SHARING
1401 /* Clear all registered shared modules. */
1402 rc = GMMR3ResetSharedModules(pVM);
1403 AssertRC(rc);
1404#endif
1405 /* Reset counters. */
1406 pVM->pgm.s.cReusedSharedPages = 0;
1407 pVM->pgm.s.cBalloonedPages = 0;
1408
1409 /*
1410 * We batch up pages that should be freed instead of calling GMM for
1411 * each and every one of them.
1412 */
1413 uint32_t cPendingPages = 0;
1414 PGMMFREEPAGESREQ pReq;
1415 rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1416 AssertLogRelRCReturn(rc, rc);
1417
1418 /*
1419 * Walk the ram ranges.
1420 */
1421 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
1422 {
1423 uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1424 AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1425
1426 if (!pVM->pgm.s.fRamPreAlloc)
1427 {
1428 /* Replace all RAM pages by ZERO pages. */
1429 while (iPage-- > 0)
1430 {
1431 PPGMPAGE pPage = &pRam->aPages[iPage];
1432 switch (PGM_PAGE_GET_TYPE(pPage))
1433 {
1434 case PGMPAGETYPE_RAM:
1435 /* Do not replace pages part of a 2 MB continuous range with zero pages, but zero them instead. */
1436 if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
1437 {
1438 void *pvPage;
1439 rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1440 AssertLogRelRCReturn(rc, rc);
1441 ASMMemZeroPage(pvPage);
1442 }
1443 else
1444 if (PGM_PAGE_IS_BALLOONED(pPage))
1445 {
1446 /* Turn into a zero page; the balloon status is lost when the VM reboots. */
1447 PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
1448 }
1449 else
1450 if (!PGM_PAGE_IS_ZERO(pPage))
1451 {
1452 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1453 AssertLogRelRCReturn(rc, rc);
1454 }
1455 break;
1456
1457 case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1458 pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1459 break;
1460
1461 case PGMPAGETYPE_MMIO2:
1462 case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
1463 case PGMPAGETYPE_ROM:
1464 case PGMPAGETYPE_MMIO:
1465 break;
1466 default:
1467 AssertFailed();
1468 }
1469 } /* for each page */
1470 }
1471 else
1472 {
1473 /* Zero the memory. */
1474 while (iPage-- > 0)
1475 {
1476 PPGMPAGE pPage = &pRam->aPages[iPage];
1477 switch (PGM_PAGE_GET_TYPE(pPage))
1478 {
1479 case PGMPAGETYPE_RAM:
1480 switch (PGM_PAGE_GET_STATE(pPage))
1481 {
1482 case PGM_PAGE_STATE_ZERO:
1483 break;
1484
1485 case PGM_PAGE_STATE_BALLOONED:
1486 /* Turn into a zero page; the balloon status is lost when the VM reboots. */
1487 PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
1488 break;
1489
1490 case PGM_PAGE_STATE_SHARED:
1491 case PGM_PAGE_STATE_WRITE_MONITORED:
1492 rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1493 AssertLogRelRCReturn(rc, rc);
1494 /* no break */
1495
1496 case PGM_PAGE_STATE_ALLOCATED:
1497 {
1498 void *pvPage;
1499 rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1500 AssertLogRelRCReturn(rc, rc);
1501 ASMMemZeroPage(pvPage);
1502 break;
1503 }
1504 }
1505 break;
1506
1507 case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1508 pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1509 break;
1510
1511 case PGMPAGETYPE_MMIO2:
1512 case PGMPAGETYPE_ROM_SHADOW:
1513 case PGMPAGETYPE_ROM:
1514 case PGMPAGETYPE_MMIO:
1515 break;
1516 default:
1517 AssertFailed();
1518
1519 }
1520 } /* for each page */
1521 }
1522
1523 }
1524
1525 /*
1526 * Finish off any pages pending freeing.
1527 */
1528 if (cPendingPages)
1529 {
1530 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
1531 AssertLogRelRCReturn(rc, rc);
1532 }
1533 GMMR3FreePagesCleanup(pReq);
1534
1535 return VINF_SUCCESS;
1536}
1537
1538/**
1539 * Frees all RAM during VM termination
1540 *
1541 * ASSUMES that the caller owns the PGM lock.
1542 *
1543 * @returns VBox status code.
1544 * @param pVM Pointer to the shared VM structure.
1545 */
1546int pgmR3PhysRamTerm(PVM pVM)
1547{
1548 Assert(PGMIsLockOwner(pVM));
1549
1550 /* Reset the memory balloon. */
1551 int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1552 AssertRC(rc);
1553
1554#ifdef VBOX_WITH_PAGE_SHARING
1555 /* Clear all registered shared modules. */
1556 rc = GMMR3ResetSharedModules(pVM);
1557 AssertRC(rc);
1558#endif
1559
1560 /*
1561 * We batch up pages that should be freed instead of calling GMM for
1562 * each and every one of them.
1563 */
1564 uint32_t cPendingPages = 0;
1565 PGMMFREEPAGESREQ pReq;
1566 rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1567 AssertLogRelRCReturn(rc, rc);
1568
1569 /*
1570 * Walk the ram ranges.
1571 */
1572 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
1573 {
1574 uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1575 AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1576
1577 /* Replace all RAM pages by ZERO pages. */
1578 while (iPage-- > 0)
1579 {
1580 PPGMPAGE pPage = &pRam->aPages[iPage];
1581 switch (PGM_PAGE_GET_TYPE(pPage))
1582 {
1583 case PGMPAGETYPE_RAM:
1584 /* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
1585 if (PGM_PAGE_IS_SHARED(pPage))
1586 {
1587 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1588 AssertLogRelRCReturn(rc, rc);
1589 }
1590 break;
1591
1592 case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1593 case PGMPAGETYPE_MMIO2:
1594 case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
1595 case PGMPAGETYPE_ROM:
1596 case PGMPAGETYPE_MMIO:
1597 break;
1598 default:
1599 AssertFailed();
1600 }
1601 } /* for each page */
1602 }
1603
1604 /*
1605 * Finish off any pages pending freeing.
1606 */
1607 if (cPendingPages)
1608 {
1609 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
1610 AssertLogRelRCReturn(rc, rc);
1611 }
1612 GMMR3FreePagesCleanup(pReq);
1613 return VINF_SUCCESS;
1614}
1615
1616/**
1617 * This is the interface IOM is using to register an MMIO region.
1618 *
1619 * It will check for conflicts and ensure that a RAM range structure
1620 * is present before calling the PGMR3HandlerPhysicalRegister API to
1621 * register the callbacks.
1622 *
1623 * @returns VBox status code.
1624 *
1625 * @param pVM Pointer to the shared VM structure.
1626 * @param GCPhys The start of the MMIO region.
1627 * @param cb The size of the MMIO region.
1628 * @param pfnHandlerR3 The address of the ring-3 handler. (IOMR3MMIOHandler)
1629 * @param pvUserR3 The user argument for R3.
1630 * @param pfnHandlerR0 The address of the ring-0 handler. (IOMMMIOHandler)
1631 * @param pvUserR0 The user argument for R0.
1632 * @param pfnHandlerRC The address of the RC handler. (IOMMMIOHandler)
1633 * @param pvUserRC The user argument for RC.
1634 * @param pszDesc The description of the MMIO region.
1635 */
1636VMMR3DECL(int) PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb,
1637 R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3, RTR3PTR pvUserR3,
1638 R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0, RTR0PTR pvUserR0,
1639 RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC, RTRCPTR pvUserRC,
1640 R3PTRTYPE(const char *) pszDesc)
1641{
1642 /*
1643 * Assert on some assumption.
1644 */
1645 VM_ASSERT_EMT(pVM);
1646 AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
1647 AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
1648 AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1649 AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
1650
1651 /*
1652 * Make sure there's a RAM range structure for the region.
1653 */
1654 int rc;
1655 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1656 bool fRamExists = false;
1657 PPGMRAMRANGE pRamPrev = NULL;
1658 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
1659 while (pRam && GCPhysLast >= pRam->GCPhys)
1660 {
1661 if ( GCPhysLast >= pRam->GCPhys
1662 && GCPhys <= pRam->GCPhysLast)
1663 {
1664 /* Simplification: all within the same range. */
1665 AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
1666 && GCPhysLast <= pRam->GCPhysLast,
1667 ("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
1668 GCPhys, GCPhysLast, pszDesc,
1669 pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
1670 VERR_PGM_RAM_CONFLICT);
1671
1672 /* Check that it's all RAM or MMIO pages. */
1673 PCPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
1674 uint32_t cLeft = cb >> PAGE_SHIFT;
1675 while (cLeft-- > 0)
1676 {
1677 AssertLogRelMsgReturn( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1678 || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO,
1679 ("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
1680 GCPhys, GCPhysLast, pszDesc, PGM_PAGE_GET_TYPE(pPage), pRam->pszDesc),
1681 VERR_PGM_RAM_CONFLICT);
1682 pPage++;
1683 }
1684
1685 /* Looks good. */
1686 fRamExists = true;
1687 break;
1688 }
1689
1690 /* next */
1691 pRamPrev = pRam;
1692 pRam = pRam->pNextR3;
1693 }
1694 PPGMRAMRANGE pNew;
1695 if (fRamExists)
1696 {
1697 pNew = NULL;
1698
1699 /*
1700 * Make all the pages in the range MMIO/ZERO pages, freeing any
1701 * RAM pages currently mapped here. This might not be 100% correct
1702 * for PCI memory, but we're doing the same thing for MMIO2 pages.
1703 */
1704 rc = pgmLock(pVM);
1705 if (RT_SUCCESS(rc))
1706 {
1707 rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
1708 pgmUnlock(pVM);
1709 }
1710 AssertRCReturn(rc, rc);
1711
1712 /* Force a PGM pool flush as guest ram references have been changed. */
1713 /** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
1714 PVMCPU pVCpu = VMMGetCpu(pVM);
1715 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
1716 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
1717 }
1718 else
1719 {
1720 pgmLock(pVM);
1721
1722 /*
1723 * No RAM range, insert an ad hoc one.
1724 *
1725 * Note that we don't have to tell REM about this range because
1726 * PGMHandlerPhysicalRegisterEx will do that for us.
1727 */
1728 Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
1729
1730 const uint32_t cPages = cb >> PAGE_SHIFT;
1731 const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
1732 rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), 16, MM_TAG_PGM_PHYS, (void **)&pNew);
1733 AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
1734
1735 /* Initialize the range. */
1736 pNew->pSelfR0 = MMHyperCCToR0(pVM, pNew);
1737 pNew->pSelfRC = MMHyperCCToRC(pVM, pNew);
1738 pNew->GCPhys = GCPhys;
1739 pNew->GCPhysLast = GCPhysLast;
1740 pNew->cb = cb;
1741 pNew->pszDesc = pszDesc;
1742 pNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO;
1743 pNew->pvR3 = NULL;
1744 pNew->paLSPages = NULL;
1745
1746 uint32_t iPage = cPages;
1747 while (iPage-- > 0)
1748 PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
1749 Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
1750
1751 /* update the page count stats. */
1752 pVM->pgm.s.cPureMmioPages += cPages;
1753 pVM->pgm.s.cAllPages += cPages;
1754
1755 /* link it */
1756 pgmR3PhysLinkRamRange(pVM, pNew, pRamPrev);
1757
1758 pgmUnlock(pVM);
1759 }
1760
1761 /*
1762 * Register the access handler.
1763 */
1764 rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_MMIO, GCPhys, GCPhysLast,
1765 pfnHandlerR3, pvUserR3,
1766 pfnHandlerR0, pvUserR0,
1767 pfnHandlerRC, pvUserRC, pszDesc);
1768 if ( RT_FAILURE(rc)
1769 && !fRamExists)
1770 {
1771 pVM->pgm.s.cPureMmioPages -= cb >> PAGE_SHIFT;
1772 pVM->pgm.s.cAllPages -= cb >> PAGE_SHIFT;
1773
1774 /* remove the ad hoc range. */
1775 pgmR3PhysUnlinkRamRange2(pVM, pNew, pRamPrev);
1776 pNew->cb = pNew->GCPhys = pNew->GCPhysLast = NIL_RTGCPHYS;
1777 MMHyperFree(pVM, pRam);
1778 }
1779 PGMPhysInvalidatePageMapTLB(pVM);
1780
1781 return rc;
1782}
1783
1784
1785/**
1786 * This is the interface IOM is using to register an MMIO region.
1787 *
1788 * It will take care of calling PGMHandlerPhysicalDeregister and clean up
1789 * any ad hoc PGMRAMRANGE left behind.
1790 *
1791 * @returns VBox status code.
1792 * @param pVM Pointer to the shared VM structure.
1793 * @param GCPhys The start of the MMIO region.
1794 * @param cb The size of the MMIO region.
1795 */
1796VMMR3DECL(int) PGMR3PhysMMIODeregister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
1797{
1798 VM_ASSERT_EMT(pVM);
1799
1800 /*
1801 * First deregister the handler, then check if we should remove the ram range.
1802 */
1803 int rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
1804 if (RT_SUCCESS(rc))
1805 {
1806 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1807 PPGMRAMRANGE pRamPrev = NULL;
1808 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
1809 while (pRam && GCPhysLast >= pRam->GCPhys)
1810 {
1811 /** @todo We're being a bit too careful here. rewrite. */
1812 if ( GCPhysLast == pRam->GCPhysLast
1813 && GCPhys == pRam->GCPhys)
1814 {
1815 Assert(pRam->cb == cb);
1816
1817 /*
1818 * See if all the pages are dead MMIO pages.
1819 */
1820 uint32_t const cPages = cb >> PAGE_SHIFT;
1821 bool fAllMMIO = true;
1822 uint32_t iPage = 0;
1823 uint32_t cLeft = cPages;
1824 while (cLeft-- > 0)
1825 {
1826 PPGMPAGE pPage = &pRam->aPages[iPage];
1827 if ( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO
1828 /*|| not-out-of-action later */)
1829 {
1830 fAllMMIO = false;
1831 Assert(PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO2_ALIAS_MMIO);
1832 AssertMsgFailed(("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
1833 break;
1834 }
1835 Assert(PGM_PAGE_IS_ZERO(pPage));
1836 pPage++;
1837 }
1838 if (fAllMMIO)
1839 {
1840 /*
1841 * Ad-hoc range, unlink and free it.
1842 */
1843 Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
1844 GCPhys, GCPhysLast, pRam->pszDesc));
1845
1846 pVM->pgm.s.cAllPages -= cPages;
1847 pVM->pgm.s.cPureMmioPages -= cPages;
1848
1849 pgmR3PhysUnlinkRamRange2(pVM, pRam, pRamPrev);
1850 pRam->cb = pRam->GCPhys = pRam->GCPhysLast = NIL_RTGCPHYS;
1851 MMHyperFree(pVM, pRam);
1852 break;
1853 }
1854 }
1855
1856 /*
1857 * Range match? It will all be within one range (see PGMAllHandler.cpp).
1858 */
1859 if ( GCPhysLast >= pRam->GCPhys
1860 && GCPhys <= pRam->GCPhysLast)
1861 {
1862 Assert(GCPhys >= pRam->GCPhys);
1863 Assert(GCPhysLast <= pRam->GCPhysLast);
1864
1865 /*
1866 * Turn the pages back into RAM pages.
1867 */
1868 uint32_t iPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
1869 uint32_t cLeft = cb >> PAGE_SHIFT;
1870 while (cLeft--)
1871 {
1872 PPGMPAGE pPage = &pRam->aPages[iPage];
1873 AssertMsg(PGM_PAGE_IS_MMIO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
1874 AssertMsg(PGM_PAGE_IS_ZERO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
1875 if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO)
1876 PGM_PAGE_SET_TYPE(pPage, PGMPAGETYPE_RAM);
1877 }
1878 break;
1879 }
1880
1881 /* next */
1882 pRamPrev = pRam;
1883 pRam = pRam->pNextR3;
1884 }
1885 }
1886
1887 /* Force a PGM pool flush as guest ram references have been changed. */
1888 /** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
1889 PVMCPU pVCpu = VMMGetCpu(pVM);
1890 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
1891 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
1892
1893 PGMPhysInvalidatePageMapTLB(pVM);
1894 return rc;
1895}
1896
1897
1898/**
1899 * Locate a MMIO2 range.
1900 *
1901 * @returns Pointer to the MMIO2 range.
1902 * @param pVM Pointer to the shared VM structure.
1903 * @param pDevIns The device instance owning the region.
1904 * @param iRegion The region.
1905 */
1906DECLINLINE(PPGMMMIO2RANGE) pgmR3PhysMMIO2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
1907{
1908 /*
1909 * Search the list.
1910 */
1911 for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
1912 if ( pCur->pDevInsR3 == pDevIns
1913 && pCur->iRegion == iRegion)
1914 return pCur;
1915 return NULL;
1916}
1917
1918
1919/**
1920 * Allocate and register an MMIO2 region.
1921 *
1922 * As mentioned elsewhere, MMIO2 is just RAM spelled differently. It's RAM
1923 * associated with a device. It is also non-shared memory with a permanent
1924 * ring-3 mapping and page backing (presently).
1925 *
1926 * A MMIO2 range may overlap with base memory if a lot of RAM is configured for
1927 * the VM, in which case we'll drop the base memory pages. Presently we will
1928 * make no attempt to preserve anything that happens to be present in the base
1929 * memory that is replaced, this is of course incorrectly but it's too much
1930 * effort.
1931 *
1932 * @returns VBox status code.
1933 * @retval VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
1934 * memory.
1935 * @retval VERR_ALREADY_EXISTS if the region already exists.
1936 *
1937 * @param pVM Pointer to the shared VM structure.
1938 * @param pDevIns The device instance owning the region.
1939 * @param iRegion The region number. If the MMIO2 memory is a PCI
1940 * I/O region this number has to be the number of that
1941 * region. Otherwise it can be any number safe
1942 * UINT8_MAX.
1943 * @param cb The size of the region. Must be page aligned.
1944 * @param fFlags Reserved for future use, must be zero.
1945 * @param ppv Where to store the pointer to the ring-3 mapping of
1946 * the memory.
1947 * @param pszDesc The description.
1948 */
1949VMMR3DECL(int) PGMR3PhysMMIO2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS cb, uint32_t fFlags, void **ppv, const char *pszDesc)
1950{
1951 /*
1952 * Validate input.
1953 */
1954 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1955 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
1956 AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
1957 AssertPtrReturn(ppv, VERR_INVALID_POINTER);
1958 AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1959 AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
1960 AssertReturn(pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion) == NULL, VERR_ALREADY_EXISTS);
1961 AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
1962 AssertReturn(cb, VERR_INVALID_PARAMETER);
1963 AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
1964
1965 const uint32_t cPages = cb >> PAGE_SHIFT;
1966 AssertLogRelReturn(((RTGCPHYS)cPages << PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
1967 AssertLogRelReturn(cPages <= INT32_MAX / 2, VERR_NO_MEMORY);
1968
1969 /*
1970 * For the 2nd+ instance, mangle the description string so it's unique.
1971 */
1972 if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
1973 {
1974 pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
1975 if (!pszDesc)
1976 return VERR_NO_MEMORY;
1977 }
1978
1979 /*
1980 * Try reserve and allocate the backing memory first as this is what is
1981 * most likely to fail.
1982 */
1983 int rc = MMR3AdjustFixedReservation(pVM, cPages, pszDesc);
1984 if (RT_SUCCESS(rc))
1985 {
1986 void *pvPages;
1987 PSUPPAGE paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(SUPPAGE));
1988 if (RT_SUCCESS(rc))
1989 rc = SUPR3PageAllocEx(cPages, 0 /*fFlags*/, &pvPages, NULL /*pR0Ptr*/, paPages);
1990 if (RT_SUCCESS(rc))
1991 {
1992 memset(pvPages, 0, cPages * PAGE_SIZE);
1993
1994 /*
1995 * Create the MMIO2 range record for it.
1996 */
1997 const size_t cbRange = RT_OFFSETOF(PGMMMIO2RANGE, RamRange.aPages[cPages]);
1998 PPGMMMIO2RANGE pNew;
1999 rc = MMR3HyperAllocOnceNoRel(pVM, cbRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
2000 AssertLogRelMsgRC(rc, ("cbRamRange=%zu\n", cbRange));
2001 if (RT_SUCCESS(rc))
2002 {
2003 pNew->pDevInsR3 = pDevIns;
2004 pNew->pvR3 = pvPages;
2005 //pNew->pNext = NULL;
2006 //pNew->fMapped = false;
2007 //pNew->fOverlapping = false;
2008 pNew->iRegion = iRegion;
2009 pNew->idSavedState = UINT8_MAX;
2010 pNew->RamRange.pSelfR0 = MMHyperCCToR0(pVM, &pNew->RamRange);
2011 pNew->RamRange.pSelfRC = MMHyperCCToRC(pVM, &pNew->RamRange);
2012 pNew->RamRange.GCPhys = NIL_RTGCPHYS;
2013 pNew->RamRange.GCPhysLast = NIL_RTGCPHYS;
2014 pNew->RamRange.pszDesc = pszDesc;
2015 pNew->RamRange.cb = cb;
2016 pNew->RamRange.fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2;
2017 pNew->RamRange.pvR3 = pvPages;
2018 //pNew->RamRange.paLSPages = NULL;
2019
2020 uint32_t iPage = cPages;
2021 while (iPage-- > 0)
2022 {
2023 PGM_PAGE_INIT(&pNew->RamRange.aPages[iPage],
2024 paPages[iPage].Phys, NIL_GMM_PAGEID,
2025 PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
2026 }
2027
2028 /* update page count stats */
2029 pVM->pgm.s.cAllPages += cPages;
2030 pVM->pgm.s.cPrivatePages += cPages;
2031
2032 /*
2033 * Link it into the list.
2034 * Since there is no particular order, just push it.
2035 */
2036 pgmLock(pVM);
2037 pNew->pNextR3 = pVM->pgm.s.pMmio2RangesR3;
2038 pVM->pgm.s.pMmio2RangesR3 = pNew;
2039 pgmUnlock(pVM);
2040
2041 *ppv = pvPages;
2042 RTMemTmpFree(paPages);
2043 PGMPhysInvalidatePageMapTLB(pVM);
2044 return VINF_SUCCESS;
2045 }
2046
2047 SUPR3PageFreeEx(pvPages, cPages);
2048 }
2049 RTMemTmpFree(paPages);
2050 MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pszDesc);
2051 }
2052 if (pDevIns->iInstance > 0)
2053 MMR3HeapFree((void *)pszDesc);
2054 return rc;
2055}
2056
2057
2058/**
2059 * Deregisters and frees an MMIO2 region.
2060 *
2061 * Any physical (and virtual) access handlers registered for the region must
2062 * be deregistered before calling this function.
2063 *
2064 * @returns VBox status code.
2065 * @param pVM Pointer to the shared VM structure.
2066 * @param pDevIns The device instance owning the region.
2067 * @param iRegion The region. If it's UINT32_MAX it'll be a wildcard match.
2068 */
2069VMMR3DECL(int) PGMR3PhysMMIO2Deregister(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2070{
2071 /*
2072 * Validate input.
2073 */
2074 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2075 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2076 AssertReturn(iRegion <= UINT8_MAX || iRegion == UINT32_MAX, VERR_INVALID_PARAMETER);
2077
2078 pgmLock(pVM);
2079 int rc = VINF_SUCCESS;
2080 unsigned cFound = 0;
2081 PPGMMMIO2RANGE pPrev = NULL;
2082 PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3;
2083 while (pCur)
2084 {
2085 if ( pCur->pDevInsR3 == pDevIns
2086 && ( iRegion == UINT32_MAX
2087 || pCur->iRegion == iRegion))
2088 {
2089 cFound++;
2090
2091 /*
2092 * Unmap it if it's mapped.
2093 */
2094 if (pCur->fMapped)
2095 {
2096 int rc2 = PGMR3PhysMMIO2Unmap(pVM, pCur->pDevInsR3, pCur->iRegion, pCur->RamRange.GCPhys);
2097 AssertRC(rc2);
2098 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2099 rc = rc2;
2100 }
2101
2102 /*
2103 * Unlink it
2104 */
2105 PPGMMMIO2RANGE pNext = pCur->pNextR3;
2106 if (pPrev)
2107 pPrev->pNextR3 = pNext;
2108 else
2109 pVM->pgm.s.pMmio2RangesR3 = pNext;
2110 pCur->pNextR3 = NULL;
2111
2112 /*
2113 * Free the memory.
2114 */
2115 int rc2 = SUPR3PageFreeEx(pCur->pvR3, pCur->RamRange.cb >> PAGE_SHIFT);
2116 AssertRC(rc2);
2117 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2118 rc = rc2;
2119
2120 uint32_t const cPages = pCur->RamRange.cb >> PAGE_SHIFT;
2121 rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pCur->RamRange.pszDesc);
2122 AssertRC(rc2);
2123 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2124 rc = rc2;
2125
2126 /* we're leaking hyper memory here if done at runtime. */
2127#ifdef VBOX_STRICT
2128 VMSTATE const enmState = VMR3GetState(pVM);
2129 AssertMsg( enmState == VMSTATE_POWERING_OFF
2130 || enmState == VMSTATE_POWERING_OFF_LS
2131 || enmState == VMSTATE_OFF
2132 || enmState == VMSTATE_OFF_LS
2133 || enmState == VMSTATE_DESTROYING
2134 || enmState == VMSTATE_TERMINATED
2135 || enmState == VMSTATE_CREATING
2136 , ("%s\n", VMR3GetStateName(enmState)));
2137#endif
2138 /*rc = MMHyperFree(pVM, pCur);
2139 AssertRCReturn(rc, rc); - not safe, see the alloc call. */
2140
2141
2142 /* update page count stats */
2143 pVM->pgm.s.cAllPages -= cPages;
2144 pVM->pgm.s.cPrivatePages -= cPages;
2145
2146 /* next */
2147 pCur = pNext;
2148 }
2149 else
2150 {
2151 pPrev = pCur;
2152 pCur = pCur->pNextR3;
2153 }
2154 }
2155 PGMPhysInvalidatePageMapTLB(pVM);
2156 pgmUnlock(pVM);
2157 return !cFound && iRegion != UINT32_MAX ? VERR_NOT_FOUND : rc;
2158}
2159
2160
2161/**
2162 * Maps a MMIO2 region.
2163 *
2164 * This is done when a guest / the bios / state loading changes the
2165 * PCI config. The replacing of base memory has the same restrictions
2166 * as during registration, of course.
2167 *
2168 * @returns VBox status code.
2169 *
2170 * @param pVM Pointer to the shared VM structure.
2171 * @param pDevIns The
2172 */
2173VMMR3DECL(int) PGMR3PhysMMIO2Map(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2174{
2175 /*
2176 * Validate input
2177 */
2178 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2179 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2180 AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2181 AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2182 AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2183 AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2184
2185 PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2186 AssertReturn(pCur, VERR_NOT_FOUND);
2187 AssertReturn(!pCur->fMapped, VERR_WRONG_ORDER);
2188 Assert(pCur->RamRange.GCPhys == NIL_RTGCPHYS);
2189 Assert(pCur->RamRange.GCPhysLast == NIL_RTGCPHYS);
2190
2191 const RTGCPHYS GCPhysLast = GCPhys + pCur->RamRange.cb - 1;
2192 AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2193
2194 /*
2195 * Find our location in the ram range list, checking for
2196 * restriction we don't bother implementing yet (partially overlapping).
2197 */
2198 bool fRamExists = false;
2199 PPGMRAMRANGE pRamPrev = NULL;
2200 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
2201 while (pRam && GCPhysLast >= pRam->GCPhys)
2202 {
2203 if ( GCPhys <= pRam->GCPhysLast
2204 && GCPhysLast >= pRam->GCPhys)
2205 {
2206 /* completely within? */
2207 AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
2208 && GCPhysLast <= pRam->GCPhysLast,
2209 ("%RGp-%RGp (MMIO2/%s) falls partly outside %RGp-%RGp (%s)\n",
2210 GCPhys, GCPhysLast, pCur->RamRange.pszDesc,
2211 pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2212 VERR_PGM_RAM_CONFLICT);
2213 fRamExists = true;
2214 break;
2215 }
2216
2217 /* next */
2218 pRamPrev = pRam;
2219 pRam = pRam->pNextR3;
2220 }
2221 if (fRamExists)
2222 {
2223 PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2224 uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2225 while (cPagesLeft-- > 0)
2226 {
2227 AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
2228 ("%RGp isn't a RAM page (%d) - mapping %RGp-%RGp (MMIO2/%s).\n",
2229 GCPhys, PGM_PAGE_GET_TYPE(pPage), GCPhys, GCPhysLast, pCur->RamRange.pszDesc),
2230 VERR_PGM_RAM_CONFLICT);
2231 pPage++;
2232 }
2233 }
2234 Log(("PGMR3PhysMMIO2Map: %RGp-%RGp fRamExists=%RTbool %s\n",
2235 GCPhys, GCPhysLast, fRamExists, pCur->RamRange.pszDesc));
2236
2237 /*
2238 * Make the changes.
2239 */
2240 pgmLock(pVM);
2241
2242 pCur->RamRange.GCPhys = GCPhys;
2243 pCur->RamRange.GCPhysLast = GCPhysLast;
2244 pCur->fMapped = true;
2245 pCur->fOverlapping = fRamExists;
2246
2247 if (fRamExists)
2248 {
2249/** @todo use pgmR3PhysFreePageRange here. */
2250 uint32_t cPendingPages = 0;
2251 PGMMFREEPAGESREQ pReq;
2252 int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2253 AssertLogRelRCReturn(rc, rc);
2254
2255 /* replace the pages, freeing all present RAM pages. */
2256 PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2257 PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2258 uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2259 while (cPagesLeft-- > 0)
2260 {
2261 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
2262 AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
2263
2264 RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
2265 PGM_PAGE_SET_HCPHYS(pPageDst, HCPhys);
2266 PGM_PAGE_SET_TYPE(pPageDst, PGMPAGETYPE_MMIO2);
2267 PGM_PAGE_SET_STATE(pPageDst, PGM_PAGE_STATE_ALLOCATED);
2268 PGM_PAGE_SET_PDE_TYPE(pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
2269 PGM_PAGE_SET_PTE_INDEX(pPageDst, 0);
2270 PGM_PAGE_SET_TRACKING(pPageDst, 0);
2271
2272 pVM->pgm.s.cZeroPages--;
2273 GCPhys += PAGE_SIZE;
2274 pPageSrc++;
2275 pPageDst++;
2276 }
2277
2278 /* Flush physical page map TLB. */
2279 PGMPhysInvalidatePageMapTLB(pVM);
2280
2281 if (cPendingPages)
2282 {
2283 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2284 AssertLogRelRCReturn(rc, rc);
2285 }
2286 GMMR3FreePagesCleanup(pReq);
2287
2288 /* Force a PGM pool flush as guest ram references have been changed. */
2289 /** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
2290 PVMCPU pVCpu = VMMGetCpu(pVM);
2291 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
2292 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2293
2294 pgmUnlock(pVM);
2295 }
2296 else
2297 {
2298 RTGCPHYS cb = pCur->RamRange.cb;
2299
2300 /* Clear the tracking data of pages we're going to reactivate. */
2301 PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2302 uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2303 while (cPagesLeft-- > 0)
2304 {
2305 PGM_PAGE_SET_TRACKING(pPageSrc, 0);
2306 PGM_PAGE_SET_PTE_INDEX(pPageSrc, 0);
2307 pPageSrc++;
2308 }
2309
2310 /* link in the ram range */
2311 pgmR3PhysLinkRamRange(pVM, &pCur->RamRange, pRamPrev);
2312 pgmUnlock(pVM);
2313
2314 REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_MMIO2);
2315 }
2316
2317 PGMPhysInvalidatePageMapTLB(pVM);
2318 return VINF_SUCCESS;
2319}
2320
2321
2322/**
2323 * Unmaps a MMIO2 region.
2324 *
2325 * This is done when a guest / the bios / state loading changes the
2326 * PCI config. The replacing of base memory has the same restrictions
2327 * as during registration, of course.
2328 */
2329VMMR3DECL(int) PGMR3PhysMMIO2Unmap(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2330{
2331 /*
2332 * Validate input
2333 */
2334 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2335 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2336 AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2337 AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2338 AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2339 AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2340
2341 PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2342 AssertReturn(pCur, VERR_NOT_FOUND);
2343 AssertReturn(pCur->fMapped, VERR_WRONG_ORDER);
2344 AssertReturn(pCur->RamRange.GCPhys == GCPhys, VERR_INVALID_PARAMETER);
2345 Assert(pCur->RamRange.GCPhysLast != NIL_RTGCPHYS);
2346
2347 Log(("PGMR3PhysMMIO2Unmap: %RGp-%RGp %s\n",
2348 pCur->RamRange.GCPhys, pCur->RamRange.GCPhysLast, pCur->RamRange.pszDesc));
2349
2350 /*
2351 * Unmap it.
2352 */
2353 pgmLock(pVM);
2354
2355 RTGCPHYS GCPhysRangeREM;
2356 RTGCPHYS cbRangeREM;
2357 bool fInformREM;
2358 if (pCur->fOverlapping)
2359 {
2360 /* Restore the RAM pages we've replaced. */
2361 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
2362 while (pRam->GCPhys > pCur->RamRange.GCPhysLast)
2363 pRam = pRam->pNextR3;
2364
2365 PPGMPAGE pPageDst = &pRam->aPages[(pCur->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2366 uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2367 while (cPagesLeft-- > 0)
2368 {
2369 PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
2370 pVM->pgm.s.cZeroPages++;
2371 pPageDst++;
2372 }
2373
2374 /* Flush physical page map TLB. */
2375 PGMPhysInvalidatePageMapTLB(pVM);
2376
2377 GCPhysRangeREM = NIL_RTGCPHYS; /* shuts up gcc */
2378 cbRangeREM = RTGCPHYS_MAX; /* ditto */
2379 fInformREM = false;
2380 }
2381 else
2382 {
2383 GCPhysRangeREM = pCur->RamRange.GCPhys;
2384 cbRangeREM = pCur->RamRange.cb;
2385 fInformREM = true;
2386
2387 pgmR3PhysUnlinkRamRange(pVM, &pCur->RamRange);
2388 }
2389
2390 pCur->RamRange.GCPhys = NIL_RTGCPHYS;
2391 pCur->RamRange.GCPhysLast = NIL_RTGCPHYS;
2392 pCur->fOverlapping = false;
2393 pCur->fMapped = false;
2394
2395 /* Force a PGM pool flush as guest ram references have been changed. */
2396 /** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
2397 PVMCPU pVCpu = VMMGetCpu(pVM);
2398 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
2399 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2400
2401 PGMPhysInvalidatePageMapTLB(pVM);
2402 pgmUnlock(pVM);
2403
2404 if (fInformREM)
2405 REMR3NotifyPhysRamDeregister(pVM, GCPhysRangeREM, cbRangeREM);
2406
2407 return VINF_SUCCESS;
2408}
2409
2410
2411/**
2412 * Checks if the given address is an MMIO2 base address or not.
2413 *
2414 * @returns true/false accordingly.
2415 * @param pVM Pointer to the shared VM structure.
2416 * @param pDevIns The owner of the memory, optional.
2417 * @param GCPhys The address to check.
2418 */
2419VMMR3DECL(bool) PGMR3PhysMMIO2IsBase(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
2420{
2421 /*
2422 * Validate input
2423 */
2424 VM_ASSERT_EMT_RETURN(pVM, false);
2425 AssertPtrReturn(pDevIns, false);
2426 AssertReturn(GCPhys != NIL_RTGCPHYS, false);
2427 AssertReturn(GCPhys != 0, false);
2428 AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), false);
2429
2430 /*
2431 * Search the list.
2432 */
2433 pgmLock(pVM);
2434 for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2435 if (pCur->RamRange.GCPhys == GCPhys)
2436 {
2437 Assert(pCur->fMapped);
2438 pgmUnlock(pVM);
2439 return true;
2440 }
2441 pgmUnlock(pVM);
2442 return false;
2443}
2444
2445
2446/**
2447 * Gets the HC physical address of a page in the MMIO2 region.
2448 *
2449 * This is API is intended for MMHyper and shouldn't be called
2450 * by anyone else...
2451 *
2452 * @returns VBox status code.
2453 * @param pVM Pointer to the shared VM structure.
2454 * @param pDevIns The owner of the memory, optional.
2455 * @param iRegion The region.
2456 * @param off The page expressed an offset into the MMIO2 region.
2457 * @param pHCPhys Where to store the result.
2458 */
2459VMMR3DECL(int) PGMR3PhysMMIO2GetHCPhys(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, PRTHCPHYS pHCPhys)
2460{
2461 /*
2462 * Validate input
2463 */
2464 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2465 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2466 AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2467
2468 pgmLock(pVM);
2469 PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2470 AssertReturn(pCur, VERR_NOT_FOUND);
2471 AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2472
2473 PCPGMPAGE pPage = &pCur->RamRange.aPages[off >> PAGE_SHIFT];
2474 *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2475 pgmUnlock(pVM);
2476 return VINF_SUCCESS;
2477}
2478
2479
2480/**
2481 * Maps a portion of an MMIO2 region into kernel space (host).
2482 *
2483 * The kernel mapping will become invalid when the MMIO2 memory is deregistered
2484 * or the VM is terminated.
2485 *
2486 * @return VBox status code.
2487 *
2488 * @param pVM Pointer to the shared VM structure.
2489 * @param pDevIns The device owning the MMIO2 memory.
2490 * @param iRegion The region.
2491 * @param off The offset into the region. Must be page aligned.
2492 * @param cb The number of bytes to map. Must be page aligned.
2493 * @param pszDesc Mapping description.
2494 * @param pR0Ptr Where to store the R0 address.
2495 */
2496VMMR3DECL(int) PGMR3PhysMMIO2MapKernel(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
2497 const char *pszDesc, PRTR0PTR pR0Ptr)
2498{
2499 /*
2500 * Validate input.
2501 */
2502 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2503 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2504 AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2505
2506 PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2507 AssertReturn(pCur, VERR_NOT_FOUND);
2508 AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2509 AssertReturn(cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2510 AssertReturn(off + cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2511
2512 /*
2513 * Pass the request on to the support library/driver.
2514 */
2515 int rc = SUPR3PageMapKernel(pCur->pvR3, off, cb, 0, pR0Ptr);
2516
2517 return rc;
2518}
2519
2520
2521/**
2522 * Registers a ROM image.
2523 *
2524 * Shadowed ROM images requires double the amount of backing memory, so,
2525 * don't use that unless you have to. Shadowing of ROM images is process
2526 * where we can select where the reads go and where the writes go. On real
2527 * hardware the chipset provides means to configure this. We provide
2528 * PGMR3PhysProtectROM() for this purpose.
2529 *
2530 * A read-only copy of the ROM image will always be kept around while we
2531 * will allocate RAM pages for the changes on demand (unless all memory
2532 * is configured to be preallocated).
2533 *
2534 * @returns VBox status.
2535 * @param pVM VM Handle.
2536 * @param pDevIns The device instance owning the ROM.
2537 * @param GCPhys First physical address in the range.
2538 * Must be page aligned!
2539 * @param cbRange The size of the range (in bytes).
2540 * Must be page aligned!
2541 * @param pvBinary Pointer to the binary data backing the ROM image.
2542 * This must be exactly \a cbRange in size.
2543 * @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
2544 * and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
2545 * @param pszDesc Pointer to description string. This must not be freed.
2546 *
2547 * @remark There is no way to remove the rom, automatically on device cleanup or
2548 * manually from the device yet. This isn't difficult in any way, it's
2549 * just not something we expect to be necessary for a while.
2550 */
2551VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
2552 const void *pvBinary, uint32_t fFlags, const char *pszDesc)
2553{
2554 Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p fFlags=%#x pszDesc=%s\n",
2555 pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, fFlags, pszDesc));
2556
2557 /*
2558 * Validate input.
2559 */
2560 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2561 AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
2562 AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
2563 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2564 AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2565 AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
2566 AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2567 AssertReturn(!(fFlags & ~(PGMPHYS_ROM_FLAGS_SHADOWED | PGMPHYS_ROM_FLAGS_PERMANENT_BINARY)), VERR_INVALID_PARAMETER);
2568 VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
2569
2570 const uint32_t cPages = cb >> PAGE_SHIFT;
2571
2572 /*
2573 * Find the ROM location in the ROM list first.
2574 */
2575 PPGMROMRANGE pRomPrev = NULL;
2576 PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3;
2577 while (pRom && GCPhysLast >= pRom->GCPhys)
2578 {
2579 if ( GCPhys <= pRom->GCPhysLast
2580 && GCPhysLast >= pRom->GCPhys)
2581 AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
2582 GCPhys, GCPhysLast, pszDesc,
2583 pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
2584 VERR_PGM_RAM_CONFLICT);
2585 /* next */
2586 pRomPrev = pRom;
2587 pRom = pRom->pNextR3;
2588 }
2589
2590 /*
2591 * Find the RAM location and check for conflicts.
2592 *
2593 * Conflict detection is a bit different than for RAM
2594 * registration since a ROM can be located within a RAM
2595 * range. So, what we have to check for is other memory
2596 * types (other than RAM that is) and that we don't span
2597 * more than one RAM range (layz).
2598 */
2599 bool fRamExists = false;
2600 PPGMRAMRANGE pRamPrev = NULL;
2601 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
2602 while (pRam && GCPhysLast >= pRam->GCPhys)
2603 {
2604 if ( GCPhys <= pRam->GCPhysLast
2605 && GCPhysLast >= pRam->GCPhys)
2606 {
2607 /* completely within? */
2608 AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
2609 && GCPhysLast <= pRam->GCPhysLast,
2610 ("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
2611 GCPhys, GCPhysLast, pszDesc,
2612 pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2613 VERR_PGM_RAM_CONFLICT);
2614 fRamExists = true;
2615 break;
2616 }
2617
2618 /* next */
2619 pRamPrev = pRam;
2620 pRam = pRam->pNextR3;
2621 }
2622 if (fRamExists)
2623 {
2624 PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2625 uint32_t cPagesLeft = cPages;
2626 while (cPagesLeft-- > 0)
2627 {
2628 AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
2629 ("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
2630 pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << PAGE_SHIFT),
2631 pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
2632 Assert(PGM_PAGE_IS_ZERO(pPage));
2633 pPage++;
2634 }
2635 }
2636
2637 /*
2638 * Update the base memory reservation if necessary.
2639 */
2640 uint32_t cExtraBaseCost = fRamExists ? 0 : cPages;
2641 if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
2642 cExtraBaseCost += cPages;
2643 if (cExtraBaseCost)
2644 {
2645 int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
2646 if (RT_FAILURE(rc))
2647 return rc;
2648 }
2649
2650 /*
2651 * Allocate memory for the virgin copy of the RAM.
2652 */
2653 PGMMALLOCATEPAGESREQ pReq;
2654 int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
2655 AssertRCReturn(rc, rc);
2656
2657 for (uint32_t iPage = 0; iPage < cPages; iPage++)
2658 {
2659 pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
2660 pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
2661 pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
2662 }
2663
2664 pgmLock(pVM);
2665 rc = GMMR3AllocatePagesPerform(pVM, pReq);
2666 pgmUnlock(pVM);
2667 if (RT_FAILURE(rc))
2668 {
2669 GMMR3AllocatePagesCleanup(pReq);
2670 return rc;
2671 }
2672
2673 /*
2674 * Allocate the new ROM range and RAM range (if necessary).
2675 */
2676 PPGMROMRANGE pRomNew;
2677 rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
2678 if (RT_SUCCESS(rc))
2679 {
2680 PPGMRAMRANGE pRamNew = NULL;
2681 if (!fRamExists)
2682 rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), sizeof(PGMPAGE), MM_TAG_PGM_PHYS, (void **)&pRamNew);
2683 if (RT_SUCCESS(rc))
2684 {
2685 pgmLock(pVM);
2686
2687 /*
2688 * Initialize and insert the RAM range (if required).
2689 */
2690 PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
2691 if (!fRamExists)
2692 {
2693 pRamNew->pSelfR0 = MMHyperCCToR0(pVM, pRamNew);
2694 pRamNew->pSelfRC = MMHyperCCToRC(pVM, pRamNew);
2695 pRamNew->GCPhys = GCPhys;
2696 pRamNew->GCPhysLast = GCPhysLast;
2697 pRamNew->cb = cb;
2698 pRamNew->pszDesc = pszDesc;
2699 pRamNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_ROM;
2700 pRamNew->pvR3 = NULL;
2701 pRamNew->paLSPages = NULL;
2702
2703 PPGMPAGE pPage = &pRamNew->aPages[0];
2704 for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
2705 {
2706 PGM_PAGE_INIT(pPage,
2707 pReq->aPages[iPage].HCPhysGCPhys,
2708 pReq->aPages[iPage].idPage,
2709 PGMPAGETYPE_ROM,
2710 PGM_PAGE_STATE_ALLOCATED);
2711
2712 pRomPage->Virgin = *pPage;
2713 }
2714
2715 pVM->pgm.s.cAllPages += cPages;
2716 pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
2717 }
2718 else
2719 {
2720 PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2721 for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
2722 {
2723 PGM_PAGE_SET_TYPE(pPage, PGMPAGETYPE_ROM);
2724 PGM_PAGE_SET_HCPHYS(pPage, pReq->aPages[iPage].HCPhysGCPhys);
2725 PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
2726 PGM_PAGE_SET_PAGEID(pPage, pReq->aPages[iPage].idPage);
2727 PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
2728 PGM_PAGE_SET_PTE_INDEX(pPage, 0);
2729 PGM_PAGE_SET_TRACKING(pPage, 0);
2730
2731 pRomPage->Virgin = *pPage;
2732 }
2733
2734 pRamNew = pRam;
2735
2736 pVM->pgm.s.cZeroPages -= cPages;
2737 }
2738 pVM->pgm.s.cPrivatePages += cPages;
2739
2740 /* Flush physical page map TLB. */
2741 PGMPhysInvalidatePageMapTLB(pVM);
2742
2743 pgmUnlock(pVM);
2744
2745
2746 /*
2747 * !HACK ALERT! REM + (Shadowed) ROM ==> mess.
2748 *
2749 * If it's shadowed we'll register the handler after the ROM notification
2750 * so we get the access handler callbacks that we should. If it isn't
2751 * shadowed we'll do it the other way around to make REM use the built-in
2752 * ROM behavior and not the handler behavior (which is to route all access
2753 * to PGM atm).
2754 */
2755 if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
2756 {
2757 REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, true /* fShadowed */);
2758 rc = PGMR3HandlerPhysicalRegister(pVM,
2759 fFlags & PGMPHYS_ROM_FLAGS_SHADOWED
2760 ? PGMPHYSHANDLERTYPE_PHYSICAL_ALL
2761 : PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
2762 GCPhys, GCPhysLast,
2763 pgmR3PhysRomWriteHandler, pRomNew,
2764 NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
2765 NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
2766 }
2767 else
2768 {
2769 rc = PGMR3HandlerPhysicalRegister(pVM,
2770 fFlags & PGMPHYS_ROM_FLAGS_SHADOWED
2771 ? PGMPHYSHANDLERTYPE_PHYSICAL_ALL
2772 : PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
2773 GCPhys, GCPhysLast,
2774 pgmR3PhysRomWriteHandler, pRomNew,
2775 NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
2776 NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
2777 REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, false /* fShadowed */);
2778 }
2779 if (RT_SUCCESS(rc))
2780 {
2781 pgmLock(pVM);
2782
2783 /*
2784 * Copy the image over to the virgin pages.
2785 * This must be done after linking in the RAM range.
2786 */
2787 PPGMPAGE pRamPage = &pRamNew->aPages[(GCPhys - pRamNew->GCPhys) >> PAGE_SHIFT];
2788 for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
2789 {
2790 void *pvDstPage;
2791 rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << PAGE_SHIFT), &pvDstPage);
2792 if (RT_FAILURE(rc))
2793 {
2794 VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
2795 break;
2796 }
2797 memcpy(pvDstPage, (const uint8_t *)pvBinary + (iPage << PAGE_SHIFT), PAGE_SIZE);
2798 }
2799 if (RT_SUCCESS(rc))
2800 {
2801 /*
2802 * Initialize the ROM range.
2803 * Note that the Virgin member of the pages has already been initialized above.
2804 */
2805 pRomNew->GCPhys = GCPhys;
2806 pRomNew->GCPhysLast = GCPhysLast;
2807 pRomNew->cb = cb;
2808 pRomNew->fFlags = fFlags;
2809 pRomNew->idSavedState = UINT8_MAX;
2810#ifdef VBOX_STRICT
2811 pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
2812 ? pvBinary : RTMemDup(pvBinary, cPages * PAGE_SIZE);
2813#else
2814 pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY ? pvBinary : NULL;
2815#endif
2816 pRomNew->pszDesc = pszDesc;
2817
2818 for (unsigned iPage = 0; iPage < cPages; iPage++)
2819 {
2820 PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
2821 pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
2822 PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
2823 }
2824
2825 /* update the page count stats for the shadow pages. */
2826 if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
2827 {
2828 pVM->pgm.s.cZeroPages += cPages;
2829 pVM->pgm.s.cAllPages += cPages;
2830 }
2831
2832 /*
2833 * Insert the ROM range, tell REM and return successfully.
2834 */
2835 pRomNew->pNextR3 = pRom;
2836 pRomNew->pNextR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
2837 pRomNew->pNextRC = pRom ? MMHyperCCToRC(pVM, pRom) : NIL_RTRCPTR;
2838
2839 if (pRomPrev)
2840 {
2841 pRomPrev->pNextR3 = pRomNew;
2842 pRomPrev->pNextR0 = MMHyperCCToR0(pVM, pRomNew);
2843 pRomPrev->pNextRC = MMHyperCCToRC(pVM, pRomNew);
2844 }
2845 else
2846 {
2847 pVM->pgm.s.pRomRangesR3 = pRomNew;
2848 pVM->pgm.s.pRomRangesR0 = MMHyperCCToR0(pVM, pRomNew);
2849 pVM->pgm.s.pRomRangesRC = MMHyperCCToRC(pVM, pRomNew);
2850 }
2851
2852 PGMPhysInvalidatePageMapTLB(pVM);
2853 GMMR3AllocatePagesCleanup(pReq);
2854 pgmUnlock(pVM);
2855 return VINF_SUCCESS;
2856 }
2857
2858 /* bail out */
2859
2860 pgmUnlock(pVM);
2861 int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
2862 AssertRC(rc2);
2863 pgmLock(pVM);
2864 }
2865
2866 if (!fRamExists)
2867 {
2868 pgmR3PhysUnlinkRamRange2(pVM, pRamNew, pRamPrev);
2869 MMHyperFree(pVM, pRamNew);
2870 }
2871 }
2872 MMHyperFree(pVM, pRomNew);
2873 }
2874
2875 /** @todo Purge the mapping cache or something... */
2876 GMMR3FreeAllocatedPages(pVM, pReq);
2877 GMMR3AllocatePagesCleanup(pReq);
2878 pgmUnlock(pVM);
2879 return rc;
2880}
2881
2882
2883/**
2884 * \#PF Handler callback for ROM write accesses.
2885 *
2886 * @returns VINF_SUCCESS if the handler have carried out the operation.
2887 * @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
2888 * @param pVM VM Handle.
2889 * @param GCPhys The physical address the guest is writing to.
2890 * @param pvPhys The HC mapping of that address.
2891 * @param pvBuf What the guest is reading/writing.
2892 * @param cbBuf How much it's reading/writing.
2893 * @param enmAccessType The access type.
2894 * @param pvUser User argument.
2895 */
2896static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf,
2897 PGMACCESSTYPE enmAccessType, void *pvUser)
2898{
2899 PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
2900 const uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
2901 Assert(iPage < (pRom->cb >> PAGE_SHIFT));
2902 PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
2903 Log5(("pgmR3PhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
2904
2905 if (enmAccessType == PGMACCESSTYPE_READ)
2906 {
2907 switch (pRomPage->enmProt)
2908 {
2909 /*
2910 * Take the default action.
2911 */
2912 case PGMROMPROT_READ_ROM_WRITE_IGNORE:
2913 case PGMROMPROT_READ_RAM_WRITE_IGNORE:
2914 case PGMROMPROT_READ_ROM_WRITE_RAM:
2915 case PGMROMPROT_READ_RAM_WRITE_RAM:
2916 return VINF_PGM_HANDLER_DO_DEFAULT;
2917
2918 default:
2919 AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
2920 pRom->aPages[iPage].enmProt, iPage, GCPhys),
2921 VERR_INTERNAL_ERROR);
2922 }
2923 }
2924 else
2925 {
2926 Assert(enmAccessType == PGMACCESSTYPE_WRITE);
2927 switch (pRomPage->enmProt)
2928 {
2929 /*
2930 * Ignore writes.
2931 */
2932 case PGMROMPROT_READ_ROM_WRITE_IGNORE:
2933 case PGMROMPROT_READ_RAM_WRITE_IGNORE:
2934 return VINF_SUCCESS;
2935
2936 /*
2937 * Write to the RAM page.
2938 */
2939 case PGMROMPROT_READ_ROM_WRITE_RAM:
2940 case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's *way* simpler that way. */
2941 {
2942 /* This should be impossible now, pvPhys doesn't work cross page anylonger. */
2943 Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> PAGE_SHIFT) == iPage);
2944
2945 /*
2946 * Take the lock, do lazy allocation, map the page and copy the data.
2947 *
2948 * Note that we have to bypass the mapping TLB since it works on
2949 * guest physical addresses and entering the shadow page would
2950 * kind of screw things up...
2951 */
2952 int rc = pgmLock(pVM);
2953 AssertRC(rc);
2954
2955 PPGMPAGE pShadowPage = &pRomPage->Shadow;
2956 if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
2957 {
2958 pShadowPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
2959 AssertLogRelReturn(pShadowPage, VERR_INTERNAL_ERROR);
2960 }
2961
2962 void *pvDstPage;
2963 rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
2964 if (RT_SUCCESS(rc))
2965 {
2966 memcpy((uint8_t *)pvDstPage + (GCPhys & PAGE_OFFSET_MASK), pvBuf, cbBuf);
2967 pRomPage->LiveSave.fWrittenTo = true;
2968 }
2969
2970 pgmUnlock(pVM);
2971 return rc;
2972 }
2973
2974 default:
2975 AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
2976 pRom->aPages[iPage].enmProt, iPage, GCPhys),
2977 VERR_INTERNAL_ERROR);
2978 }
2979 }
2980}
2981
2982
2983/**
2984 * Called by PGMR3Reset to reset the shadow, switch to the virgin,
2985 * and verify that the virgin part is untouched.
2986 *
2987 * This is done after the normal memory has been cleared.
2988 *
2989 * ASSUMES that the caller owns the PGM lock.
2990 *
2991 * @param pVM The VM handle.
2992 */
2993int pgmR3PhysRomReset(PVM pVM)
2994{
2995 Assert(PGMIsLockOwner(pVM));
2996 for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
2997 {
2998 const uint32_t cPages = pRom->cb >> PAGE_SHIFT;
2999
3000 if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3001 {
3002 /*
3003 * Reset the physical handler.
3004 */
3005 int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
3006 AssertRCReturn(rc, rc);
3007
3008 /*
3009 * What we do with the shadow pages depends on the memory
3010 * preallocation option. If not enabled, we'll just throw
3011 * out all the dirty pages and replace them by the zero page.
3012 */
3013 if (!pVM->pgm.s.fRamPreAlloc)
3014 {
3015 /* Free the dirty pages. */
3016 uint32_t cPendingPages = 0;
3017 PGMMFREEPAGESREQ pReq;
3018 rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
3019 AssertRCReturn(rc, rc);
3020
3021 for (uint32_t iPage = 0; iPage < cPages; iPage++)
3022 if ( !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
3023 && !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
3024 {
3025 Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
3026 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
3027 pRom->GCPhys + (iPage << PAGE_SHIFT));
3028 AssertLogRelRCReturn(rc, rc);
3029 }
3030
3031 if (cPendingPages)
3032 {
3033 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
3034 AssertLogRelRCReturn(rc, rc);
3035 }
3036 GMMR3FreePagesCleanup(pReq);
3037 }
3038 else
3039 {
3040 /* clear all the shadow pages. */
3041 for (uint32_t iPage = 0; iPage < cPages; iPage++)
3042 {
3043 Assert(!PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow));
3044 Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
3045 void *pvDstPage;
3046 const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3047 rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
3048 if (RT_FAILURE(rc))
3049 break;
3050 ASMMemZeroPage(pvDstPage);
3051 }
3052 AssertRCReturn(rc, rc);
3053 }
3054 }
3055
3056#ifdef VBOX_STRICT
3057 /*
3058 * Verify that the virgin page is unchanged if possible.
3059 */
3060 if (pRom->pvOriginal)
3061 {
3062 uint8_t const *pbSrcPage = (uint8_t const *)pRom->pvOriginal;
3063 for (uint32_t iPage = 0; iPage < cPages; iPage++, pbSrcPage += PAGE_SIZE)
3064 {
3065 const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3066 void const *pvDstPage;
3067 int rc = pgmPhysPageMapReadOnly(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPage);
3068 if (RT_FAILURE(rc))
3069 break;
3070 if (memcmp(pvDstPage, pbSrcPage, PAGE_SIZE))
3071 LogRel(("pgmR3PhysRomReset: %RGp rom page changed (%s) - loaded saved state?\n",
3072 GCPhys, pRom->pszDesc));
3073 }
3074 }
3075#endif
3076 }
3077
3078 return VINF_SUCCESS;
3079}
3080
3081
3082/**
3083 * Called by PGMR3Term to free resources.
3084 *
3085 * ASSUMES that the caller owns the PGM lock.
3086 *
3087 * @param pVM The VM handle.
3088 */
3089void pgmR3PhysRomTerm(PVM pVM)
3090{
3091#ifdef RT_STRICT
3092 /*
3093 * Free the heap copy of the original bits.
3094 */
3095 for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3096 {
3097 if ( pRom->pvOriginal
3098 && !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
3099 {
3100 RTMemFree((void *)pRom->pvOriginal);
3101 pRom->pvOriginal = NULL;
3102 }
3103 }
3104#endif
3105}
3106
3107
3108/**
3109 * Change the shadowing of a range of ROM pages.
3110 *
3111 * This is intended for implementing chipset specific memory registers
3112 * and will not be very strict about the input. It will silently ignore
3113 * any pages that are not the part of a shadowed ROM.
3114 *
3115 * @returns VBox status code.
3116 * @retval VINF_PGM_SYNC_CR3
3117 *
3118 * @param pVM Pointer to the shared VM structure.
3119 * @param GCPhys Where to start. Page aligned.
3120 * @param cb How much to change. Page aligned.
3121 * @param enmProt The new ROM protection.
3122 */
3123VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
3124{
3125 /*
3126 * Check input
3127 */
3128 if (!cb)
3129 return VINF_SUCCESS;
3130 AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3131 AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3132 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
3133 AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3134 AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
3135
3136 /*
3137 * Process the request.
3138 */
3139 pgmLock(pVM);
3140 int rc = VINF_SUCCESS;
3141 bool fFlushTLB = false;
3142 for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3143 {
3144 if ( GCPhys <= pRom->GCPhysLast
3145 && GCPhysLast >= pRom->GCPhys
3146 && (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
3147 {
3148 /*
3149 * Iterate the relevant pages and make necessary the changes.
3150 */
3151 bool fChanges = false;
3152 uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
3153 ? pRom->cb >> PAGE_SHIFT
3154 : (GCPhysLast - pRom->GCPhys + 1) >> PAGE_SHIFT;
3155 for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3156 iPage < cPages;
3157 iPage++)
3158 {
3159 PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3160 if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
3161 {
3162 fChanges = true;
3163
3164 /* flush references to the page. */
3165 PPGMPAGE pRamPage = pgmPhysGetPage(&pVM->pgm.s, pRom->GCPhys + (iPage << PAGE_SHIFT));
3166 int rc2 = pgmPoolTrackFlushGCPhys(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT), pRamPage, &fFlushTLB);
3167 if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS || RT_FAILURE(rc2)))
3168 rc = rc2;
3169
3170 PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
3171 PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
3172
3173 *pOld = *pRamPage;
3174 *pRamPage = *pNew;
3175 /** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
3176 }
3177 pRomPage->enmProt = enmProt;
3178 }
3179
3180 /*
3181 * Reset the access handler if we made changes, no need
3182 * to optimize this.
3183 */
3184 if (fChanges)
3185 {
3186 int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
3187 if (RT_FAILURE(rc2))
3188 {
3189 pgmUnlock(pVM);
3190 AssertRC(rc);
3191 return rc2;
3192 }
3193 }
3194
3195 /* Advance - cb isn't updated. */
3196 GCPhys = pRom->GCPhys + (cPages << PAGE_SHIFT);
3197 }
3198 }
3199 pgmUnlock(pVM);
3200 if (fFlushTLB)
3201 PGM_INVL_ALL_VCPU_TLBS(pVM);
3202
3203 return rc;
3204}
3205
3206
3207/**
3208 * Sets the Address Gate 20 state.
3209 *
3210 * @param pVCpu The VCPU to operate on.
3211 * @param fEnable True if the gate should be enabled.
3212 * False if the gate should be disabled.
3213 */
3214VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
3215{
3216 LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
3217 if (pVCpu->pgm.s.fA20Enabled != fEnable)
3218 {
3219 pVCpu->pgm.s.fA20Enabled = fEnable;
3220 pVCpu->pgm.s.GCPhysA20Mask = ~(RTGCPHYS)(!fEnable << 20);
3221 REMR3A20Set(pVCpu->pVMR3, pVCpu, fEnable);
3222 /** @todo we're not handling this correctly for VT-x / AMD-V. See #2911 */
3223 }
3224}
3225
3226#ifdef PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST
3227/**
3228 * Tree enumeration callback for dealing with age rollover.
3229 * It will perform a simple compression of the current age.
3230 */
3231static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
3232{
3233 Assert(PGMIsLockOwner((PVM)pvUser));
3234 /* Age compression - ASSUMES iNow == 4. */
3235 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3236 if (pChunk->iAge >= UINT32_C(0xffffff00))
3237 pChunk->iAge = 3;
3238 else if (pChunk->iAge >= UINT32_C(0xfffff000))
3239 pChunk->iAge = 2;
3240 else if (pChunk->iAge)
3241 pChunk->iAge = 1;
3242 else /* iAge = 0 */
3243 pChunk->iAge = 4;
3244 return 0;
3245}
3246
3247
3248/**
3249 * Tree enumeration callback that updates the chunks that have
3250 * been used since the last
3251 */
3252static DECLCALLBACK(int) pgmR3PhysChunkAgeingCallback(PAVLU32NODECORE pNode, void *pvUser)
3253{
3254 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3255 if (!pChunk->iAge)
3256 {
3257 PVM pVM = (PVM)pvUser;
3258 pChunk->iAge = pVM->pgm.s.ChunkR3Map.iNow;
3259 }
3260 return 0;
3261}
3262
3263
3264/**
3265 * Performs ageing of the ring-3 chunk mappings.
3266 *
3267 * @param pVM The VM handle.
3268 */
3269VMMR3DECL(void) PGMR3PhysChunkAgeing(PVM pVM)
3270{
3271 pgmLock(pVM);
3272 pVM->pgm.s.ChunkR3Map.AgeingCountdown = RT_MIN(pVM->pgm.s.ChunkR3Map.cMax / 4, 1024);
3273 pVM->pgm.s.ChunkR3Map.iNow++;
3274 if (pVM->pgm.s.ChunkR3Map.iNow == 0)
3275 {
3276 pVM->pgm.s.ChunkR3Map.iNow = 4;
3277 RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkAgeingRolloverCallback, pVM);
3278 }
3279 else
3280 RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkAgeingCallback, pVM);
3281 pgmUnlock(pVM);
3282}
3283
3284
3285/**
3286 * The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
3287 */
3288typedef struct PGMR3PHYSCHUNKUNMAPCB
3289{
3290 PVM pVM; /**< The VM handle. */
3291 PPGMCHUNKR3MAP pChunk; /**< The chunk to unmap. */
3292 uint32_t iLastAge; /**< Highest age found so far. */
3293} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
3294
3295
3296/**
3297 * Callback used to find the mapping that's been unused for
3298 * the longest time.
3299 */
3300static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
3301{
3302 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3303 PPGMR3PHYSCHUNKUNMAPCB pArg = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
3304
3305 if ( pChunk->iAge
3306 && !pChunk->cRefs
3307 && pArg->iLastAge < pChunk->iAge)
3308 {
3309 /*
3310 * Check that it's not in any of the TLBs.
3311 */
3312 PVM pVM = pArg->pVM;
3313 for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
3314 if (pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk == pChunk)
3315 {
3316 pChunk = NULL;
3317 break;
3318 }
3319 if (pChunk)
3320 for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
3321 if (pVM->pgm.s.PhysTlbHC.aEntries[i].pMap == pChunk)
3322 {
3323 pChunk = NULL;
3324 break;
3325 }
3326 if (pChunk)
3327 {
3328 pArg->pChunk = pChunk;
3329 pArg->iLastAge = pChunk->iAge;
3330 }
3331 }
3332 return 0;
3333}
3334
3335
3336/**
3337 * Finds a good candidate for unmapping when the ring-3 mapping cache is full.
3338 *
3339 * The candidate will not be part of any TLBs, so no need to flush
3340 * anything afterwards.
3341 *
3342 * @returns Chunk id.
3343 * @param pVM The VM handle.
3344 */
3345static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
3346{
3347 Assert(PGMIsLockOwner(pVM));
3348
3349 /*
3350 * Do tree ageing first?
3351 */
3352 if (pVM->pgm.s.ChunkR3Map.AgeingCountdown-- == 0)
3353 {
3354 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkAging, a);
3355 PGMR3PhysChunkAgeing(pVM);
3356 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkAging, a);
3357 }
3358
3359 /*
3360 * Enumerate the age tree starting with the left most node.
3361 */
3362 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3363 PGMR3PHYSCHUNKUNMAPCB Args;
3364 Args.pVM = pVM;
3365 Args.pChunk = NULL;
3366 Args.iLastAge = 0;
3367 RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
3368 Assert(Args.pChunk);
3369 if (Args.pChunk)
3370 {
3371 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3372 return Args.pChunk->Core.Key;
3373 }
3374
3375 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3376 return INT32_MAX;
3377}
3378
3379/**
3380 * Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
3381 *
3382 * This is only called on one of the EMTs while the other ones are waiting for
3383 * it to complete this function.
3384 *
3385 * @returns VINF_SUCCESS (VBox strict status code).
3386 * @param pVM The VM handle.
3387 * @param pVCpu The VMCPU for the EMT we're being called on. Unused.
3388 * @param pvUser User pointer. Unused
3389 *
3390 */
3391DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
3392{
3393 int rc = VINF_SUCCESS;
3394 pgmLock(pVM);
3395
3396 if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
3397 {
3398 /* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
3399 /* todo: also not really efficient to unmap a chunk that contains PD or PT pages. */
3400 pgmR3PoolClearAllRendezvous(pVM, &pVM->aCpus[0], NULL /* no need to flush the REM TLB as we already did that above */);
3401
3402 /*
3403 * Request the ring-0 part to unmap a chunk to make space in the mapping cache.
3404 */
3405 GMMMAPUNMAPCHUNKREQ Req;
3406 Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
3407 Req.Hdr.cbReq = sizeof(Req);
3408 Req.pvR3 = NULL;
3409 Req.idChunkMap = NIL_GMM_CHUNKID;
3410 Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
3411
3412 if (Req.idChunkUnmap != INT32_MAX)
3413 {
3414 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
3415 rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
3416 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
3417 if (RT_SUCCESS(rc))
3418 {
3419 /* remove the unmapped one. */
3420 PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
3421 AssertRelease(pUnmappedChunk);
3422 pUnmappedChunk->pv = NULL;
3423 pUnmappedChunk->Core.Key = UINT32_MAX;
3424#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3425 MMR3HeapFree(pUnmappedChunk);
3426#else
3427 MMR3UkHeapFree(pVM, pUnmappedChunk, MM_TAG_PGM_CHUNK_MAPPING);
3428#endif
3429 pVM->pgm.s.ChunkR3Map.c--;
3430 pVM->pgm.s.cUnmappedChunks++;
3431
3432 /* Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses) */
3433 /* todo: we should not flush chunks which include cr3 mappings. */
3434 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3435 {
3436 PPGMCPU pPGM = &pVM->aCpus[idCpu].pgm.s;
3437
3438 pPGM->pGst32BitPdR3 = NULL;
3439 pPGM->pGstPaePdptR3 = NULL;
3440 pPGM->pGstAmd64Pml4R3 = NULL;
3441#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3442 pPGM->pGst32BitPdR0 = NIL_RTR0PTR;
3443 pPGM->pGstPaePdptR0 = NIL_RTR0PTR;
3444 pPGM->pGstAmd64Pml4R0 = NIL_RTR0PTR;
3445#endif
3446 for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
3447 {
3448 pPGM->apGstPaePDsR3[i] = NULL;
3449#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3450 pPGM->apGstPaePDsR0[i] = NIL_RTR0PTR;
3451#endif
3452 }
3453
3454 /* Flush REM TLBs. */
3455 CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3456 }
3457
3458 /* Flush REM translation blocks. */
3459 REMFlushTBs(pVM);
3460 }
3461 }
3462 }
3463 pgmUnlock(pVM);
3464 return rc;
3465}
3466
3467/**
3468 * Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
3469 *
3470 * @returns VBox status code.
3471 * @param pVM The VM to operate on.
3472 */
3473void pgmR3PhysUnmapChunk(PVM pVM)
3474{
3475 int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
3476 AssertRC(rc);
3477}
3478#endif /* PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST */
3479
3480/**
3481 * Maps the given chunk into the ring-3 mapping cache.
3482 *
3483 * This will call ring-0.
3484 *
3485 * @returns VBox status code.
3486 * @param pVM The VM handle.
3487 * @param idChunk The chunk in question.
3488 * @param ppChunk Where to store the chunk tracking structure.
3489 *
3490 * @remarks Called from within the PGM critical section.
3491 * @remarks Can be called from any thread!
3492 */
3493int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
3494{
3495 int rc;
3496
3497 Assert(PGMIsLockOwner(pVM));
3498 /*
3499 * Allocate a new tracking structure first.
3500 */
3501#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3502 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
3503#else
3504 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3UkHeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk), NULL);
3505#endif
3506 AssertReturn(pChunk, VERR_NO_MEMORY);
3507 pChunk->Core.Key = idChunk;
3508
3509 /*
3510 * Request the ring-0 part to map the chunk in question.
3511 */
3512 GMMMAPUNMAPCHUNKREQ Req;
3513 Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
3514 Req.Hdr.cbReq = sizeof(Req);
3515 Req.pvR3 = NULL;
3516 Req.idChunkMap = idChunk;
3517 Req.idChunkUnmap = NIL_GMM_CHUNKID;
3518
3519 /* Must be callable from any thread, so can't use VMMR3CallR0. */
3520 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
3521 rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
3522 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
3523 if (RT_SUCCESS(rc))
3524 {
3525 /*
3526 * Update the tree.
3527 */
3528 /* insert the new one. */
3529 AssertPtr(Req.pvR3);
3530 pChunk->pv = Req.pvR3;
3531 bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
3532 AssertRelease(fRc);
3533 pVM->pgm.s.ChunkR3Map.c++;
3534 pVM->pgm.s.cMappedChunks++;
3535
3536 /* If we're running out of virtual address space, then we should unmap another chunk. */
3537 if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
3538 {
3539#ifdef PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST
3540 /* Postpone the unmap operation (which requires a rendezvous operation) as we own the PGM lock here. */
3541 rc = VMR3ReqCallNoWaitU(pVM->pUVM, VMCPUID_ANY, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
3542 AssertRC(rc);
3543#else
3544 AssertFatalFailed(); /* can't happen */
3545#endif
3546 }
3547 }
3548 else
3549 {
3550 AssertRC(rc);
3551#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3552 MMR3HeapFree(pChunk);
3553#else
3554 MMR3UkHeapFree(pVM, pChunk, MM_TAG_PGM_CHUNK_MAPPING);
3555#endif
3556 pChunk = NULL;
3557 }
3558
3559 *ppChunk = pChunk;
3560 return rc;
3561}
3562
3563
3564/**
3565 * For VMMCALLRING3_PGM_MAP_CHUNK, considered internal.
3566 *
3567 * @returns see pgmR3PhysChunkMap.
3568 * @param pVM The VM handle.
3569 * @param idChunk The chunk to map.
3570 */
3571VMMR3DECL(int) PGMR3PhysChunkMap(PVM pVM, uint32_t idChunk)
3572{
3573 PPGMCHUNKR3MAP pChunk;
3574 int rc;
3575
3576 pgmLock(pVM);
3577 rc = pgmR3PhysChunkMap(pVM, idChunk, &pChunk);
3578 pgmUnlock(pVM);
3579 return rc;
3580}
3581
3582
3583/**
3584 * Invalidates the TLB for the ring-3 mapping cache.
3585 *
3586 * @param pVM The VM handle.
3587 */
3588VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
3589{
3590 pgmLock(pVM);
3591 for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
3592 {
3593 pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
3594 pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
3595 }
3596 /* The page map TLB references chunks, so invalidate that one too. */
3597 PGMPhysInvalidatePageMapTLB(pVM);
3598 pgmUnlock(pVM);
3599}
3600
3601
3602/**
3603 * Response to VMMCALLRING3_PGM_ALLOCATE_LARGE_PAGE to allocate a large (2MB) page
3604 * for use with a nested paging PDE.
3605 *
3606 * @returns The following VBox status codes.
3607 * @retval VINF_SUCCESS on success.
3608 * @retval VINF_EM_NO_MEMORY if we're out of memory.
3609 *
3610 * @param pVM The VM handle.
3611 * @param GCPhys GC physical start address of the 2 MB range
3612 */
3613VMMR3DECL(int) PGMR3PhysAllocateLargeHandyPage(PVM pVM, RTGCPHYS GCPhys)
3614{
3615 pgmLock(pVM);
3616
3617 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
3618 int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE, 0, NULL);
3619 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
3620 if (RT_SUCCESS(rc))
3621 {
3622 Assert(pVM->pgm.s.cLargeHandyPages == 1);
3623
3624 uint32_t idPage = pVM->pgm.s.aLargeHandyPage[0].idPage;
3625 RTHCPHYS HCPhys = pVM->pgm.s.aLargeHandyPage[0].HCPhysGCPhys;
3626
3627 void *pv;
3628
3629 /* Map the large page into our address space.
3630 *
3631 * Note: assuming that within the 2 MB range:
3632 * - GCPhys + PAGE_SIZE = HCPhys + PAGE_SIZE (whole point of this exercise)
3633 * - user space mapping is continuous as well
3634 * - page id (GCPhys) + 1 = page id (GCPhys + PAGE_SIZE)
3635 */
3636 rc = pgmPhysPageMapByPageID(pVM, idPage, HCPhys, &pv);
3637 AssertLogRelMsg(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", idPage, HCPhys, rc));
3638
3639 if (RT_SUCCESS(rc))
3640 {
3641 /*
3642 * Clear the pages.
3643 */
3644 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
3645 for (unsigned i = 0; i < _2M/PAGE_SIZE; i++)
3646 {
3647 ASMMemZeroPage(pv);
3648
3649 PPGMPAGE pPage;
3650 rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
3651 AssertRC(rc);
3652
3653 Assert(PGM_PAGE_IS_ZERO(pPage));
3654 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
3655 pVM->pgm.s.cZeroPages--;
3656
3657 /*
3658 * Do the PGMPAGE modifications.
3659 */
3660 pVM->pgm.s.cPrivatePages++;
3661 PGM_PAGE_SET_HCPHYS(pPage, HCPhys);
3662 PGM_PAGE_SET_PAGEID(pPage, idPage);
3663 PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
3664 PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_PDE);
3665 PGM_PAGE_SET_PTE_INDEX(pPage, 0);
3666 PGM_PAGE_SET_TRACKING(pPage, 0);
3667
3668 /* Somewhat dirty assumption that page ids are increasing. */
3669 idPage++;
3670
3671 HCPhys += PAGE_SIZE;
3672 GCPhys += PAGE_SIZE;
3673
3674 pv = (void *)((uintptr_t)pv + PAGE_SIZE);
3675
3676 Log3(("PGMR3PhysAllocateLargePage: idPage=%#x HCPhys=%RGp\n", idPage, HCPhys));
3677 }
3678 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
3679
3680 /* Flush all TLBs. */
3681 PGM_INVL_ALL_VCPU_TLBS(pVM);
3682 PGMPhysInvalidatePageMapTLB(pVM);
3683 }
3684 pVM->pgm.s.cLargeHandyPages = 0;
3685 }
3686
3687 pgmUnlock(pVM);
3688 return rc;
3689}
3690
3691
3692/**
3693 * Response to VM_FF_PGM_NEED_HANDY_PAGES and VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES.
3694 *
3695 * This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
3696 * signal and clear the out of memory condition. When contracted, this API is
3697 * used to try clear the condition when the user wants to resume.
3698 *
3699 * @returns The following VBox status codes.
3700 * @retval VINF_SUCCESS on success. FFs cleared.
3701 * @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
3702 * this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
3703 *
3704 * @param pVM The VM handle.
3705 *
3706 * @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
3707 * in EM.cpp and shouldn't be propagated outside TRPM, HWACCM, EM and
3708 * pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
3709 * handler.
3710 */
3711VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
3712{
3713 pgmLock(pVM);
3714
3715 /*
3716 * Allocate more pages, noting down the index of the first new page.
3717 */
3718 uint32_t iClear = pVM->pgm.s.cHandyPages;
3719 AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_INTERNAL_ERROR);
3720 Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
3721 int rcAlloc = VINF_SUCCESS;
3722 int rcSeed = VINF_SUCCESS;
3723 int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
3724 while (rc == VERR_GMM_SEED_ME)
3725 {
3726 void *pvChunk;
3727 rcAlloc = rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
3728 if (RT_SUCCESS(rc))
3729 {
3730 rcSeed = rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
3731 if (RT_FAILURE(rc))
3732 SUPR3PageFree(pvChunk, GMM_CHUNK_SIZE >> PAGE_SHIFT);
3733 }
3734 if (RT_SUCCESS(rc))
3735 rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
3736 }
3737
3738 /* todo: we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
3739 if ( rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
3740 && pVM->pgm.s.cHandyPages > 0)
3741 {
3742 /* Still handy pages left, so don't panic. */
3743 rc = VINF_SUCCESS;
3744 }
3745
3746 if (RT_SUCCESS(rc))
3747 {
3748 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
3749 Assert(pVM->pgm.s.cHandyPages > 0);
3750 VM_FF_CLEAR(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
3751 VM_FF_CLEAR(pVM, VM_FF_PGM_NO_MEMORY);
3752
3753 /*
3754 * Clear the pages.
3755 */
3756 while (iClear < pVM->pgm.s.cHandyPages)
3757 {
3758 PGMMPAGEDESC pPage = &pVM->pgm.s.aHandyPages[iClear];
3759 void *pv;
3760 rc = pgmPhysPageMapByPageID(pVM, pPage->idPage, pPage->HCPhysGCPhys, &pv);
3761 AssertLogRelMsgBreak(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", pPage->idPage, pPage->HCPhysGCPhys, rc));
3762 ASMMemZeroPage(pv);
3763 iClear++;
3764 Log3(("PGMR3PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
3765 }
3766 }
3767 else
3768 {
3769 uint64_t cAllocPages, cMaxPages, cBalloonPages;
3770
3771 /*
3772 * We should never get here unless there is a genuine shortage of
3773 * memory (or some internal error). Flag the error so the VM can be
3774 * suspended ASAP and the user informed. If we're totally out of
3775 * handy pages we will return failure.
3776 */
3777 /* Report the failure. */
3778 LogRel(("PGM: Failed to procure handy pages; rc=%Rrc rcAlloc=%Rrc rcSeed=%Rrc cHandyPages=%#x\n"
3779 " cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
3780 rc, rcAlloc, rcSeed,
3781 pVM->pgm.s.cHandyPages,
3782 pVM->pgm.s.cAllPages,
3783 pVM->pgm.s.cPrivatePages,
3784 pVM->pgm.s.cSharedPages,
3785 pVM->pgm.s.cZeroPages));
3786
3787 if (GMMR3QueryMemoryStats(pVM, &cAllocPages, &cMaxPages, &cBalloonPages) == VINF_SUCCESS)
3788 {
3789 LogRel(("GMM: Statistics:\n"
3790 " Allocated pages: %RX64\n"
3791 " Maximum pages: %RX64\n"
3792 " Ballooned pages: %RX64\n", cAllocPages, cMaxPages, cBalloonPages));
3793 }
3794
3795 if ( rc != VERR_NO_MEMORY
3796 && rc != VERR_LOCK_FAILED)
3797 {
3798 for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
3799 {
3800 LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
3801 i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
3802 pVM->pgm.s.aHandyPages[i].idSharedPage));
3803 uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
3804 if (idPage != NIL_GMM_PAGEID)
3805 {
3806 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
3807 pRam;
3808 pRam = pRam->pNextR3)
3809 {
3810 uint32_t const cPages = pRam->cb >> PAGE_SHIFT;
3811 for (uint32_t iPage = 0; iPage < cPages; iPage++)
3812 if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
3813 LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
3814 pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
3815 }
3816 }
3817 }
3818 }
3819
3820 /* Set the FFs and adjust rc. */
3821 VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
3822 VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
3823 if ( rc == VERR_NO_MEMORY
3824 || rc == VERR_LOCK_FAILED)
3825 rc = VINF_EM_NO_MEMORY;
3826 }
3827
3828 pgmUnlock(pVM);
3829 return rc;
3830}
3831
3832
3833/**
3834 * Frees the specified RAM page and replaces it with the ZERO page.
3835 *
3836 * This is used by ballooning, remapping MMIO2 and RAM reset.
3837 *
3838 * @param pVM Pointer to the shared VM structure.
3839 * @param pReq Pointer to the request.
3840 * @param pPage Pointer to the page structure.
3841 * @param GCPhys The guest physical address of the page, if applicable.
3842 *
3843 * @remarks The caller must own the PGM lock.
3844 */
3845int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys)
3846{
3847 /*
3848 * Assert sanity.
3849 */
3850 Assert(PGMIsLockOwner(pVM));
3851 if (RT_UNLIKELY( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
3852 && PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
3853 {
3854 AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
3855 return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
3856 }
3857
3858 if ( PGM_PAGE_IS_ZERO(pPage)
3859 || PGM_PAGE_IS_BALLOONED(pPage))
3860 return VINF_SUCCESS;
3861
3862 const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
3863 Log3(("pgmPhysFreePage: idPage=%#x HCPhys=%RGp pPage=%R[pgmpage]\n", idPage, pPage));
3864 if (RT_UNLIKELY( idPage == NIL_GMM_PAGEID
3865 || idPage > GMM_PAGEID_LAST
3866 || PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID))
3867 {
3868 AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
3869 return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
3870 }
3871
3872 /* update page count stats. */
3873 if (PGM_PAGE_IS_SHARED(pPage))
3874 pVM->pgm.s.cSharedPages--;
3875 else
3876 pVM->pgm.s.cPrivatePages--;
3877 pVM->pgm.s.cZeroPages++;
3878
3879 /* Deal with write monitored pages. */
3880 if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
3881 {
3882 PGM_PAGE_SET_WRITTEN_TO(pPage);
3883 pVM->pgm.s.cWrittenToPages++;
3884 }
3885
3886 /*
3887 * pPage = ZERO page.
3888 */
3889 PGM_PAGE_SET_HCPHYS(pPage, pVM->pgm.s.HCPhysZeroPg);
3890 PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
3891 PGM_PAGE_SET_PAGEID(pPage, NIL_GMM_PAGEID);
3892 PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
3893 PGM_PAGE_SET_PTE_INDEX(pPage, 0);
3894 PGM_PAGE_SET_TRACKING(pPage, 0);
3895
3896 /* Flush physical page map TLB entry. */
3897 PGMPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
3898
3899 /*
3900 * Make sure it's not in the handy page array.
3901 */
3902 for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
3903 {
3904 if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
3905 {
3906 pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
3907 break;
3908 }
3909 if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
3910 {
3911 pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
3912 break;
3913 }
3914 }
3915
3916 /*
3917 * Push it onto the page array.
3918 */
3919 uint32_t iPage = *pcPendingPages;
3920 Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
3921 *pcPendingPages += 1;
3922
3923 pReq->aPages[iPage].idPage = idPage;
3924
3925 if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
3926 return VINF_SUCCESS;
3927
3928 /*
3929 * Flush the pages.
3930 */
3931 int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
3932 if (RT_SUCCESS(rc))
3933 {
3934 GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
3935 *pcPendingPages = 0;
3936 }
3937 return rc;
3938}
3939
3940
3941/**
3942 * Converts a GC physical address to a HC ring-3 pointer, with some
3943 * additional checks.
3944 *
3945 * @returns VBox status code.
3946 * @retval VINF_SUCCESS on success.
3947 * @retval VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
3948 * access handler of some kind.
3949 * @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
3950 * accesses or is odd in any way.
3951 * @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
3952 *
3953 * @param pVM The VM handle.
3954 * @param GCPhys The GC physical address to convert.
3955 * @param fWritable Whether write access is required.
3956 * @param ppv Where to store the pointer corresponding to GCPhys on
3957 * success.
3958 */
3959VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
3960{
3961 pgmLock(pVM);
3962
3963 PPGMRAMRANGE pRam;
3964 PPGMPAGE pPage;
3965 int rc = pgmPhysGetPageAndRangeEx(&pVM->pgm.s, GCPhys, &pPage, &pRam);
3966 if (RT_SUCCESS(rc))
3967 {
3968 if (PGM_PAGE_IS_BALLOONED(pPage))
3969 rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
3970 else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
3971 rc = VINF_SUCCESS;
3972 else
3973 {
3974 if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
3975 rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
3976 else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
3977 {
3978 /** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
3979 * in -norawr0 mode. */
3980 if (fWritable)
3981 rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
3982 }
3983 else
3984 {
3985 /* Temporarily disabled physical handler(s), since the recompiler
3986 doesn't get notified when it's reset we'll have to pretend it's
3987 operating normally. */
3988 if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
3989 rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
3990 else
3991 rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
3992 }
3993 }
3994 if (RT_SUCCESS(rc))
3995 {
3996 int rc2;
3997
3998 /* Make sure what we return is writable. */
3999 if (fWritable)
4000 switch (PGM_PAGE_GET_STATE(pPage))
4001 {
4002 case PGM_PAGE_STATE_ALLOCATED:
4003 break;
4004 case PGM_PAGE_STATE_BALLOONED:
4005 AssertFailed();
4006 break;
4007 case PGM_PAGE_STATE_ZERO:
4008 case PGM_PAGE_STATE_SHARED:
4009 if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
4010 break;
4011 case PGM_PAGE_STATE_WRITE_MONITORED:
4012 rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
4013 AssertLogRelRCReturn(rc2, rc2);
4014 break;
4015 }
4016
4017 /* Get a ring-3 mapping of the address. */
4018 PPGMPAGER3MAPTLBE pTlbe;
4019 rc2 = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
4020 AssertLogRelRCReturn(rc2, rc2);
4021 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4022 /** @todo mapping/locking hell; this isn't horribly efficient since
4023 * pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
4024
4025 Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] *ppv=%p\n", GCPhys, rc, pPage, *ppv));
4026 }
4027 else
4028 Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
4029
4030 /* else: handler catching all access, no pointer returned. */
4031 }
4032 else
4033 rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
4034
4035 pgmUnlock(pVM);
4036 return rc;
4037}
4038
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette