VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/PGMR0DynMap.cpp@ 14446

Last change on this file since 14446 was 14404, checked in by vboxsync, 16 years ago

PGMR0DynMap: Disabled the init/term code.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 29.2 KB
Line 
1/* $Id: PGMR0DynMap.cpp 14404 2008-11-20 12:20:49Z vboxsync $ */
2/** @file
3 * PGM - Page Manager and Monitor, ring-0 dynamic mapping cache.
4 */
5
6/*
7 * Copyright (C) 2008 Sun Microsystems, Inc.
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18 * Clara, CA 95054 USA or visit http://www.sun.com if you need
19 * additional information or have any questions.
20 */
21
22/*******************************************************************************
23* Internal Functions *
24*******************************************************************************/
25#include <VBox/pgm.h>
26#include "../PGMInternal.h"
27#include <VBox/vm.h>
28#include <VBox/sup.h>
29#include <VBox/err.h>
30#include <iprt/asm.h>
31#include <iprt/alloc.h>
32#include <iprt/assert.h>
33#include <iprt/cpuset.h>
34#include <iprt/memobj.h>
35#include <iprt/mp.h>
36#include <iprt/spinlock.h>
37#include <iprt/semaphore.h>
38
39
40/*******************************************************************************
41* Structures and Typedefs *
42*******************************************************************************/
43/**
44 * Ring-0 dynamic mapping cache segment.
45 *
46 * The dynamic mapping cache can be extended with additional segments if the
47 * load is found to be too high. This done the next time a VM is created, under
48 * the protection of the init mutex. The arrays is reallocated and the new
49 * segment is added to the end of these. Nothing is rehashed of course, as the
50 * indexes / addresses must remain unchanged.
51 *
52 * This structure is only modified while owning the init mutex or during module
53 * init / term.
54 */
55typedef struct PGMR0DYNMAPSEG
56{
57 /** Pointer to the next segment. */
58 struct PGMR0DYNMAPSEG *pNext;
59 /** The memory object for the virtual address range that we're abusing. */
60 RTR0MEMOBJ hMemObj;
61 /** The memory object for the page tables. */
62 RTR0MEMOBJ hMemObjPT;
63 /** The start page in the cache. (I.e. index into the arrays.) */
64 uint32_t iPage;
65 /** The number of pages this segment contributes. */
66 uint32_t cPages;
67} PGMR0DYNMAPSEG;
68/** Pointer to a ring-0 dynamic mapping cache segment. */
69typedef PGMR0DYNMAPSEG *PPGMR0DYNMAPSEG;
70
71
72/**
73 * Ring-0 dynamic mapping cache entry.
74 *
75 * This structure tracks
76 */
77typedef struct PGMR0DYNMAPENTRY
78{
79 /** The physical address of the currently mapped page.
80 * This is duplicate for three reasons: cache locality, cache policy of the PT
81 * mappings and sanity checks. */
82 RTHCPHYS HCPhys;
83 /** Pointer to the page. */
84 void *pvPage;
85 /** The number of references. */
86 int32_t volatile cRefs;
87 /** PTE pointer union. */
88 union PGMR0DYNMAPENTRY_PPTE
89 {
90 /** PTE pointer, 32-bit legacy version. */
91 PX86PTE pLegacy;
92 /** PTE pointer, PAE version. */
93 PX86PTEPAE pPae;
94 } uPte;
95 /** CPUs that haven't invalidated this entry after it's last update. */
96 RTCPUSET PendingSet;
97} PGMR0DYNMAPENTRY;
98/** Pointer to a ring-0 dynamic mapping cache entry. */
99typedef PGMR0DYNMAPENTRY *PPGMR0DYNMAPENTRY;
100
101
102/**
103 * Ring-0 dynamic mapping cache.
104 *
105 * This is initialized during VMMR0 module init but no segments are allocated at
106 * that time. Segments will be added when the first VM is started and removed
107 * again when the last VM shuts down, thus avoid consuming memory while dormant.
108 * At module termination, the remaining bits will be freed up.
109 */
110typedef struct PGMR0DYNMAP
111{
112 /** The usual magic number / eye catcher (PGMR0DYNMAP_MAGIC). */
113 uint32_t u32Magic;
114 /** Spinlock serializing the normal operation of the cache. */
115 RTSPINLOCK hSpinlock;
116 /** Array for tracking and managing the pages. */
117 PPGMR0DYNMAPENTRY paPages;
118 /** The cache size given as a number of pages. */
119 uint32_t cPages;
120 /** Whether it's 32-bit legacy or PAE/AMD64 paging mode. */
121 bool fLegacyMode;
122 /** The current load. */
123 uint32_t cLoad;
124 /** The max load.
125 * This is maintained to get trigger adding of more mapping space. */
126 uint32_t cMaxLoad;
127 /** Initialization / termination lock. */
128 RTSEMFASTMUTEX hInitLock;
129 /** The number of users (protected by hInitLock). */
130 uint32_t cUsers;
131 /** Array containing a copy of the original page tables.
132 * The entries are either X86PTE or X86PTEPAE according to fLegacyMode. */
133 void *pvSavedPTEs;
134 /** List of segments. */
135 PPGMR0DYNMAPSEG pSegHead;
136} PGMR0DYNMAP;
137/** Pointer to the ring-0 dynamic mapping cache */
138typedef PGMR0DYNMAP *PPGMR0DYNMAP;
139
140/** PGMR0DYNMAP::u32Magic. (Jens Christian Bugge Wesseltoft) */
141#define PGMR0DYNMAP_MAGIC 0x19640201
142
143
144/*******************************************************************************
145* Global Variables *
146*******************************************************************************/
147/** Pointer to the ring-0 dynamic mapping cache. */
148static PPGMR0DYNMAP g_pPGMR0DynMap;
149
150
151/*******************************************************************************
152* Internal Functions *
153*******************************************************************************/
154static void pgmR0DynMapReleasePage(PPGMR0DYNMAP pThis, uint32_t iPage, uint32_t cRefs);
155static int pgmR0DynMapSetup(PPGMR0DYNMAP pThis);
156static int pgmR0DynMapGrow(PPGMR0DYNMAP pThis);
157static void pgmR0DynMapTearDown(PPGMR0DYNMAP pThis);
158
159
160/**
161 * Initializes the ring-0 dynamic mapping cache.
162 *
163 * @returns VBox status code.
164 */
165VMMR0DECL(int) PGMR0DynMapInit(void)
166{
167#ifndef DEBUG_bird
168 return VINF_SUCCESS;
169#else
170 Assert(!g_pPGMR0DynMap);
171
172 /*
173 * Create and initialize the cache instance.
174 */
175 PPGMR0DYNMAP pThis = (PPGMR0DYNMAP)RTMemAllocZ(sizeof(*pThis));
176 AssertLogRelReturn(pThis, VERR_NO_MEMORY);
177 int rc = VINF_SUCCESS;
178 SUPPAGINGMODE enmMode = SUPR0GetPagingMode();
179 switch (enmMode)
180 {
181 case SUPPAGINGMODE_32_BIT:
182 case SUPPAGINGMODE_32_BIT_GLOBAL:
183 pThis->fLegacyMode = false;
184 break;
185 case SUPPAGINGMODE_PAE:
186 case SUPPAGINGMODE_PAE_GLOBAL:
187 case SUPPAGINGMODE_PAE_NX:
188 case SUPPAGINGMODE_PAE_GLOBAL_NX:
189 case SUPPAGINGMODE_AMD64:
190 case SUPPAGINGMODE_AMD64_GLOBAL:
191 case SUPPAGINGMODE_AMD64_NX:
192 case SUPPAGINGMODE_AMD64_GLOBAL_NX:
193 pThis->fLegacyMode = false;
194 break;
195 default:
196 rc = VERR_INTERNAL_ERROR;
197 break;
198 }
199 if (RT_SUCCESS(rc))
200 {
201 rc = RTSemFastMutexCreate(&pThis->hInitLock);
202 if (RT_SUCCESS(rc))
203 {
204 rc = RTSpinlockCreate(&pThis->hSpinlock);
205 if (RT_SUCCESS(rc))
206 {
207 pThis->u32Magic = PGMR0DYNMAP_MAGIC;
208 g_pPGMR0DynMap = pThis;
209 return VINF_SUCCESS;
210 }
211 RTSemFastMutexDestroy(pThis->hInitLock);
212 }
213 }
214 RTMemFree(pThis);
215 return rc;
216#endif
217}
218
219
220/**
221 * Terminates the ring-0 dynamic mapping cache.
222 */
223VMMR0DECL(void) PGMR0DynMapTerm(void)
224{
225#ifdef DEBUG_bird
226 /*
227 * Destroy the cache.
228 *
229 * There is not supposed to be any races here, the loader should
230 * make sure about that. So, don't bother locking anything.
231 *
232 * The VM objects should all be destroyed by now, so there is no
233 * dangling users or anything like that to clean up. This routine
234 * is just a mirror image of PGMR0DynMapInit.
235 */
236 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
237 if (pThis)
238 {
239 AssertPtr(pThis);
240 g_pPGMR0DynMap = NULL;
241
242 AssertLogRelMsg(!pThis->cUsers && !pThis->paPages && !pThis->cPages,
243 ("cUsers=%d paPages=%p cPages=%#x\n",
244 pThis->cUsers, pThis->paPages, pThis->cPages));
245
246 /* Free the associated resources. */
247 RTSemFastMutexDestroy(pThis->hInitLock);
248 pThis->hInitLock = NIL_RTSEMFASTMUTEX;
249 RTSpinlockDestroy(pThis->hSpinlock);
250 pThis->hSpinlock = NIL_RTSPINLOCK;
251 pThis->u32Magic = UINT32_MAX;
252 RTMemFree(pThis);
253 }
254#endif
255}
256
257
258/**
259 * Initializes the dynamic mapping cache for a new VM.
260 *
261 * @returns VBox status code.
262 * @param pVM Pointer to the shared VM structure.
263 */
264VMMR0DECL(int) PGMR0DynMapInitVM(PVM pVM)
265{
266#ifndef DEBUG_bird
267 return VINF_SUCCESS;
268#else
269 /*
270 * Initialize the auto sets.
271 */
272 VMCPUID idCpu = pVM->cCPUs;
273 while (idCpu-- > 0)
274 {
275 PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
276 uint32_t j = RT_ELEMENTS(pSet->aEntries);
277 while (j-- > 0)
278 {
279 pSet->aEntries[j].iPage = UINT16_MAX;
280 pSet->aEntries[j].cRefs = 0;
281 }
282 pSet->cEntries = PGMMAPSET_CLOSED;
283 }
284
285 /*
286 * Do we need the cache? Skip the last bit if we don't.
287 */
288 Assert(!pVM->pgm.s.pvR0DynMapUsed);
289 pVM->pgm.s.pvR0DynMapUsed = NULL;
290 if (!HWACCMIsEnabled(pVM))
291 return VINF_SUCCESS;
292
293 /*
294 * Reference and if necessary setup or grow the cache.
295 */
296 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
297 AssertPtrReturn(pThis, VERR_INTERNAL_ERROR);
298 int rc = RTSemFastMutexRequest(pThis->hInitLock);
299 AssertLogRelRCReturn(rc, rc);
300
301 pThis->cUsers++;
302 if (pThis->cUsers == 1)
303 rc = pgmR0DynMapSetup(pThis);
304 else if (pThis->cMaxLoad > pThis->cPages / 2)
305 rc = pgmR0DynMapGrow(pThis);
306 if (RT_FAILURE(rc))
307 pThis->cUsers--;
308
309 RTSemFastMutexRelease(pThis->hInitLock);
310
311 return rc;
312#endif
313}
314
315
316/**
317 * Terminates the dynamic mapping cache usage for a VM.
318 *
319 * @param pVM Pointer to the shared VM structure.
320 */
321VMMR0DECL(void) PGMR0DynMapTermVM(PVM pVM)
322{
323#ifdef DEBUG_bird
324 /*
325 * Return immediately if we're not using the cache.
326 */
327 if (!pVM->pgm.s.pvR0DynMapUsed)
328 return;
329
330 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
331 AssertPtrReturnVoid(pThis);
332
333 int rc = RTSemFastMutexRequest(pThis->hInitLock);
334 AssertLogRelRCReturnVoid(rc);
335
336 if (pVM->pgm.s.pvR0DynMapUsed == pThis)
337 {
338 pVM->pgm.s.pvR0DynMapUsed = NULL;
339
340 /*
341 * Clean up and check the auto sets.
342 */
343 VMCPUID idCpu = pVM->cCPUs;
344 while (idCpu-- > 0)
345 {
346 PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
347 uint32_t j = pSet->cEntries;
348 if (j <= RT_ELEMENTS(pSet->aEntries))
349 {
350 /*
351 * The set is open, close it.
352 */
353 while (j-- > 0)
354 {
355 int32_t cRefs = pSet->aEntries[j].cRefs;
356 uint32_t iPage = pSet->aEntries[j].iPage;
357 LogRel(("PGMR0DynMapTermVM: %d dangling refs to %#x\n", cRefs, iPage));
358 if (iPage < pThis->cPages && cRefs > 0)
359 pgmR0DynMapReleasePage(pThis, iPage, cRefs);
360 else
361 AssertMsgFailed(("cRefs=%d iPage=%#x cPages=%u\n", cRefs, iPage, pThis->cPages));
362
363 pSet->aEntries[j].iPage = UINT16_MAX;
364 pSet->aEntries[j].cRefs = 0;
365 }
366 pSet->cEntries = PGMMAPSET_CLOSED;
367 }
368
369 j = RT_ELEMENTS(pSet->aEntries);
370 while (j-- > 0)
371 {
372 Assert(pSet->aEntries[j].iPage == UINT16_MAX);
373 Assert(!pSet->aEntries[j].cRefs);
374 }
375 }
376
377 /*
378 * Release our reference to the mapping cache.
379 */
380 Assert(pThis->cUsers > 0);
381 pThis->cUsers--;
382 if (!pThis->cUsers)
383 pgmR0DynMapTearDown(pThis);
384 }
385 else
386 AssertMsgFailed(("pvR0DynMapUsed=%p pThis=%p\n", pVM->pgm.s.pvR0DynMapUsed, pThis));
387
388 RTSemFastMutexRelease(pThis->hInitLock);
389#endif
390}
391
392
393/**
394 * Called by PGMR0DynMapInitVM under the init lock.
395 *
396 * @returns VBox status code.
397 * @param pThis The dynamic mapping cache instance.
398 */
399static int pgmR0DynMapSetup(PPGMR0DYNMAP pThis)
400{
401 return VINF_SUCCESS;
402}
403
404
405/**
406 * Called by PGMR0DynMapInitVM under the init lock.
407 *
408 * @returns VBox status code.
409 * @param pThis The dynamic mapping cache instance.
410 */
411static int pgmR0DynMapGrow(PPGMR0DYNMAP pThis)
412{
413 return VINF_SUCCESS;
414}
415
416
417/**
418 * Shoots down the TLBs for all the cache pages, pgmR0DynMapTearDown helper.
419 *
420 * @param idCpu The current CPU.
421 * @param pvUser1 The dynamic mapping cache instance.
422 * @param pvUser2 Unused, NULL.
423 */
424static DECLCALLBACK(void) pgmR0DynMapShootDownTlbs(RTCPUID idCpu, void *pvUser1, void *pvUser2)
425{
426 Assert(!pvUser2);
427 PPGMR0DYNMAP pThis = (PPGMR0DYNMAP)pvUser1;
428 AssertPtr(pThis == g_pPGMR0DynMap);
429 PPGMR0DYNMAPENTRY paPages = pThis->paPages;
430 uint32_t iPage = pThis->cPages;
431 while (iPage-- > 0)
432 ASMInvalidatePage(paPages[iPage].pvPage);
433}
434
435
436/**
437 * Called by PGMR0DynMapTermVM under the init lock.
438 *
439 * @returns VBox status code.
440 * @param pThis The dynamic mapping cache instance.
441 */
442static void pgmR0DynMapTearDown(PPGMR0DYNMAP pThis)
443{
444 /*
445 * Restore the original page table entries
446 */
447 PPGMR0DYNMAPENTRY paPages = pThis->paPages;
448 uint32_t iPage = pThis->cPages;
449 if (pThis->fLegacyMode)
450 {
451 X86PGUINT const *paSavedPTEs = (X86PGUINT const *)pThis->pvSavedPTEs;
452 while (iPage-- > 0)
453 {
454 X86PGUINT uOld = paPages[iPage].uPte.pLegacy->u;
455 X86PGUINT uOld2 = uOld; NOREF(uOld2);
456 X86PGUINT uNew = paSavedPTEs[iPage];
457 while (!ASMAtomicCmpXchgExU32(&paPages[iPage].uPte.pLegacy->u, uNew, uOld, &uOld))
458 AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
459 }
460 }
461 else
462 {
463 X86PGPAEUINT const *paSavedPTEs = (X86PGPAEUINT const *)pThis->pvSavedPTEs;
464 while (iPage-- > 0)
465 {
466 X86PGPAEUINT uOld = paPages[iPage].uPte.pPae->u;
467 X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
468 X86PGPAEUINT uNew = paSavedPTEs[iPage];
469 while (!ASMAtomicCmpXchgExU64(&paPages[iPage].uPte.pPae->u, uNew, uOld, &uOld))
470 AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
471 }
472 }
473
474 /*
475 * Shoot down the TLBs on all CPUs before freeing them.
476 * If RTMpOnAll fails, make sure the TLBs are invalidated on the current CPU at least.
477 */
478 int rc = RTMpOnAll(pgmR0DynMapShootDownTlbs, pThis, NULL);
479 AssertRC(rc);
480 if (RT_FAILURE(rc))
481 {
482 iPage = pThis->cPages;
483 while (iPage-- > 0)
484 ASMInvalidatePage(paPages[iPage].pvPage);
485 }
486
487 /*
488 * Free the segments.
489 */
490 while (pThis->pSegHead)
491 {
492 PPGMR0DYNMAPSEG pSeg = pThis->pSegHead;
493 pThis->pSegHead = pSeg->pNext;
494
495 int rc;
496 rc = RTR0MemObjFree(pSeg->hMemObjPT, true /* fFreeMappings */); AssertRC(rc);
497 pSeg->hMemObjPT = NIL_RTR0MEMOBJ;
498 rc = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */); AssertRC(rc);
499 pSeg->hMemObj = NIL_RTR0MEMOBJ;
500 pSeg->pNext = NULL;
501 pSeg->iPage = UINT32_MAX;
502 pSeg->cPages = 0;
503 RTMemFree(pSeg);
504 }
505
506 /*
507 * Free the arrays and restore the initial state.
508 * The cLoadMax value is left behind for the next setup.
509 */
510 RTMemFree(pThis->paPages);
511 pThis->paPages = NULL;
512 RTMemFree(pThis->pvSavedPTEs);
513 pThis->pvSavedPTEs = NULL;
514 pThis->cPages = 0;
515 pThis->cLoad = 0;
516}
517
518
519/**
520 * Release references to a page, caller owns the spin lock.
521 *
522 * @param pThis The dynamic mapping cache instance.
523 * @param iPage The page.
524 * @param cRefs The number of references to release.
525 */
526DECLINLINE(void) pgmR0DynMapReleasePageLocked(PPGMR0DYNMAP pThis, uint32_t iPage, int32_t cRefs)
527{
528 cRefs = ASMAtomicSubS32(&pThis->paPages[iPage].cRefs, cRefs);
529 AssertMsg(cRefs >= 0, ("%d\n", cRefs));
530 if (!cRefs)
531 pThis->cLoad--;
532}
533
534
535/**
536 * Release references to a page, caller does not own the spin lock.
537 *
538 * @param pThis The dynamic mapping cache instance.
539 * @param iPage The page.
540 * @param cRefs The number of references to release.
541 */
542static void pgmR0DynMapReleasePage(PPGMR0DYNMAP pThis, uint32_t iPage, uint32_t cRefs)
543{
544 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
545 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
546 pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
547 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
548}
549
550
551/**
552 * pgmR0DynMapPage worker that deals with the tedious bits.
553 *
554 * @returns The page index on success, UINT32_MAX on failure.
555 * @param pThis The dynamic mapping cache instance.
556 * @param HCPhys The address of the page to be mapped.
557 * @param iPage The page index pgmR0DynMapPage hashed HCPhys to.
558 */
559static uint32_t pgmR0DynMapPageSlow(PPGMR0DYNMAP pThis, RTHCPHYS HCPhys, uint32_t iPage)
560{
561 /*
562 * Check if any of the first 5 pages are unreferenced since the caller
563 * already has made sure they aren't matching.
564 */
565 uint32_t const cPages = cPages;
566 PPGMR0DYNMAPENTRY paPages = pThis->paPages;
567 uint32_t iFreePage;
568 if (!paPages[iPage].cRefs)
569 iFreePage = iPage;
570 else if (!paPages[(iPage + 1) % cPages].cRefs)
571 iFreePage = iPage;
572 else if (!paPages[(iPage + 2) % cPages].cRefs)
573 iFreePage = iPage;
574 else if (!paPages[(iPage + 3) % cPages].cRefs)
575 iFreePage = iPage;
576 else if (!paPages[(iPage + 4) % cPages].cRefs)
577 iFreePage = iPage;
578 else
579 {
580 /*
581 * Search for an unused or matching entry.
582 */
583 iFreePage = (iPage + 5) % pThis->cPages;
584 for (;;)
585 {
586 if (paPages[iFreePage].HCPhys == HCPhys)
587 return iFreePage;
588 if (!paPages[iFreePage].cRefs)
589 break;
590
591 /* advance */
592 iFreePage = (iFreePage + 1) % cPages;
593 if (RT_UNLIKELY(iFreePage != iPage))
594 return UINT32_MAX;
595 }
596 }
597
598 /*
599 * Setup the new entry.
600 */
601 paPages[iFreePage].HCPhys = HCPhys;
602 RTCpuSetFill(&paPages[iFreePage].PendingSet);
603 if (pThis->fLegacyMode)
604 {
605 X86PGUINT uOld = paPages[iFreePage].uPte.pLegacy->u;
606 X86PGUINT uOld2 = uOld; NOREF(uOld2);
607 X86PGUINT uNew = (uOld & X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT)
608 | X86_PTE_P | X86_PTE_A | X86_PTE_D
609 | (HCPhys & X86_PTE_PG_MASK);
610 while (!ASMAtomicCmpXchgExU32(&paPages[iFreePage].uPte.pLegacy->u, uNew, uOld, &uOld))
611 AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
612 }
613 else
614 {
615 X86PGPAEUINT uOld = paPages[iFreePage].uPte.pPae->u;
616 X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
617 X86PGPAEUINT uNew = (uOld & X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT)
618 | X86_PTE_P | X86_PTE_A | X86_PTE_D
619 | (HCPhys & X86_PTE_PAE_PG_MASK);
620 while (!ASMAtomicCmpXchgExU64(&paPages[iFreePage].uPte.pPae->u, uNew, uOld, &uOld))
621 AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
622 }
623 return iFreePage;
624}
625
626
627/**
628 * Maps a page into the pool.
629 *
630 * @returns Pointer to the mapping.
631 * @param pThis The dynamic mapping cache instance.
632 * @param HCPhys The address of the page to be mapped.
633 * @param piPage Where to store the page index.
634 */
635DECLINLINE(void *) pgmR0DynMapPage(PPGMR0DYNMAP pThis, RTHCPHYS HCPhys, uint32_t *piPage)
636{
637 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
638 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
639 AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
640
641 /*
642 * Find an entry, if possible a matching one. The HCPhys address is hashed
643 * down to a page index, collisions are handled by linear searching. Optimize
644 * for a hit in the first 5 pages.
645 *
646 * To the cheap hits here and defer the tedious searching and inserting
647 * to a helper function.
648 */
649 uint32_t const cPages = cPages;
650 uint32_t iPage = (HCPhys >> PAGE_SHIFT) % cPages;
651 PPGMR0DYNMAPENTRY paPages = pThis->paPages;
652 if (paPages[iPage].HCPhys != HCPhys)
653 {
654 uint32_t iPage2 = (iPage + 1) % cPages;
655 if (paPages[iPage2].HCPhys != HCPhys)
656 {
657 iPage2 = (iPage + 2) % cPages;
658 if (paPages[iPage2].HCPhys != HCPhys)
659 {
660 iPage2 = (iPage + 3) % cPages;
661 if (paPages[iPage2].HCPhys != HCPhys)
662 {
663 iPage2 = (iPage + 4) % cPages;
664 if (paPages[iPage2].HCPhys != HCPhys)
665 {
666 iPage = pgmR0DynMapPageSlow(pThis, HCPhys, iPage);
667 if (RT_UNLIKELY(iPage == UINT32_MAX))
668 {
669 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
670 return NULL;
671 }
672 }
673 else
674 iPage = iPage2;
675 }
676 else
677 iPage = iPage2;
678 }
679 else
680 iPage = iPage2;
681 }
682 else
683 iPage = iPage2;
684 }
685
686 /*
687 * Reference it, update statistics and get the return address.
688 */
689 if (ASMAtomicIncS32(&paPages[iPage].cRefs) == 1)
690 {
691 pThis->cLoad++;
692 if (pThis->cLoad > pThis->cMaxLoad)
693 pThis->cMaxLoad = pThis->cLoad;
694 Assert(pThis->cLoad <= pThis->cPages);
695 }
696 void *pvPage = paPages[iPage].pvPage;
697
698 /*
699 * Invalidate the entry?
700 */
701 RTCPUID idRealCpu = RTMpCpuId();
702 bool fInvalidateIt = RTCpuSetIsMember(&paPages[iPage].PendingSet, idRealCpu);
703 if (fInvalidateIt)
704 RTCpuSetDel(&paPages[iPage].PendingSet, idRealCpu);
705
706 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
707
708 /*
709 * Do the actual invalidation outside the spinlock.
710 */
711 ASMInvalidatePage(pvPage);
712
713 *piPage = iPage;
714 return pvPage;
715}
716
717
718/**
719 * Signals the start of a new set of mappings.
720 *
721 * Mostly for strictness. PGMDynMapHCPage won't work unless this
722 * API is called.
723 *
724 * @param pVCpu The shared data for the current virtual CPU.
725 */
726VMMDECL(void) PGMDynMapStartAutoSet(PVMCPU pVCpu)
727{
728 Assert(pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED);
729 pVCpu->pgm.s.AutoSet.cEntries = 0;
730}
731
732
733/**
734 * Releases the dynamic memory mappings made by PGMDynMapHCPage and associates
735 * since the PGMDynMapStartAutoSet call.
736 *
737 * @param pVCpu The shared data for the current virtual CPU.
738 */
739VMMDECL(void) PGMDynMapReleaseAutoSet(PVMCPU pVCpu)
740{
741 PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
742
743 /* close the set */
744 uint32_t i = pVCpu->pgm.s.AutoSet.cEntries;
745 AssertMsg(i <= RT_ELEMENTS(pVCpu->pgm.s.AutoSet.aEntries), ("%#x (%u)\n", i, i));
746 pVCpu->pgm.s.AutoSet.cEntries = PGMMAPSET_CLOSED;
747
748 /* release any pages we're referencing. */
749 if (i != 0 && RT_LIKELY(i <= RT_ELEMENTS(pVCpu->pgm.s.AutoSet.aEntries)))
750 {
751 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
752 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
753 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
754
755 while (i-- > 0)
756 {
757 uint32_t iPage = pSet->aEntries[i].iPage;
758 Assert(iPage < pThis->cPages);
759 int32_t cRefs = pSet->aEntries[i].cRefs;
760 Assert(cRefs > 0);
761 pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
762
763 pSet->aEntries[i].iPage = UINT16_MAX;
764 pSet->aEntries[i].cRefs = 0;
765 }
766
767 Assert(pThis->cLoad <= pThis->cPages);
768 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
769 }
770}
771
772
773/**
774 * Migrates the automatic mapping set of the current vCPU if necessary.
775 *
776 * This is called when re-entering the hardware assisted execution mode after a
777 * nip down to ring-3. We run the risk that the CPU might have change and we
778 * will therefore make sure all the cache entries currently in the auto set will
779 * be valid on the new CPU. If the cpu didn't change nothing will happen as all
780 * the entries will have been flagged as invalidated.
781 *
782 * @param pVCpu The shared data for the current virtual CPU.
783 * @thread EMT
784 */
785VMMDECL(void) PGMDynMapMigrateAutoSet(PVMCPU pVCpu)
786{
787 PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
788 uint32_t i = pVCpu->pgm.s.AutoSet.cEntries;
789 AssertMsg(i <= RT_ELEMENTS(pVCpu->pgm.s.AutoSet.aEntries), ("%#x (%u)\n", i, i));
790 if (i != 0 && RT_LIKELY(i <= RT_ELEMENTS(pVCpu->pgm.s.AutoSet.aEntries)))
791 {
792 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
793 RTCPUID idRealCpu = RTMpCpuId();
794
795 while (i-- > 0)
796 {
797 Assert(pSet->aEntries[i].cRefs > 0);
798 uint32_t iPage = pSet->aEntries[i].iPage;
799 Assert(iPage < pThis->cPages);
800 if (RTCpuSetIsMember(&pThis->paPages[iPage].PendingSet, idRealCpu))
801 {
802 RTCpuSetDel(&pThis->paPages[iPage].PendingSet, idRealCpu);
803 ASMInvalidatePage(pThis->paPages[iPage].pvPage);
804 }
805 }
806 }
807}
808
809
810/**
811 * As a final resort for a full auto set, try merge duplicate entries.
812 *
813 * @param pSet The set.
814 */
815static void pgmDynMapOptimizeAutoSet(PPGMMAPSET pSet)
816{
817 for (uint32_t i = 0 ; i < pSet->cEntries; i++)
818 {
819 uint16_t const iPage = pSet->aEntries[i].iPage;
820 uint32_t j = i + 1;
821 while (j < pSet->cEntries)
822 {
823 if (pSet->aEntries[j].iPage != iPage)
824 j++;
825 else
826 {
827 /* merge j with i removing j. */
828 pSet->aEntries[i].cRefs += pSet->aEntries[j].cRefs;
829 pSet->cEntries--;
830 if (j < pSet->cEntries)
831 {
832 pSet->aEntries[j] = pSet->aEntries[pSet->cEntries];
833 pSet->aEntries[pSet->cEntries].iPage = UINT16_MAX;
834 pSet->aEntries[pSet->cEntries].cRefs = 0;
835 }
836 else
837 {
838 pSet->aEntries[j].iPage = UINT16_MAX;
839 pSet->aEntries[j].cRefs = 0;
840 }
841 }
842 }
843 }
844}
845
846
847/* documented elsewhere - a bit of a mess. */
848VMMDECL(int) PGMDynMapHCPage(PVM pVM, RTHCPHYS HCPhys, void **ppv)
849{
850 /*
851 * Validate state.
852 */
853 AssertMsgReturn(pVM->pgm.s.pvR0DynMapUsed == g_pPGMR0DynMap,
854 ("%p != %p\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap),
855 VERR_ACCESS_DENIED);
856 AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
857 PVMCPU pVCpu = VMMGetCpu(pVM);
858 PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
859 AssertPtrReturn(pVCpu, VERR_INTERNAL_ERROR);
860 AssertMsgReturn(pSet->cEntries > RT_ELEMENTS(pSet->aEntries),
861 ("%#x (%u)\n", pSet->cEntries, pSet->cEntries), VERR_WRONG_ORDER);
862
863 /*
864 * Map it.
865 */
866 uint32_t iPage;
867 void *pvPage = pgmR0DynMapPage(g_pPGMR0DynMap, HCPhys, &iPage);
868 if (RT_UNLIKELY(!pvPage))
869 {
870 static uint32_t s_cBitched = 0;
871 if (++s_cBitched < 10)
872 LogRel(("PGMDynMapHCPage: cLoad=%u/%u cPages=%u\n",
873 g_pPGMR0DynMap->cLoad, g_pPGMR0DynMap->cMaxLoad, g_pPGMR0DynMap->cPages));
874 return VERR_PGM_DYNMAP_FAILED;
875 }
876
877 /*
878 * Add the page to the auto reference set.
879 * If it's less than half full, don't bother looking for duplicates.
880 */
881 if (pSet->cEntries < RT_ELEMENTS(pSet->aEntries) / 2)
882 {
883 pSet->aEntries[pSet->cEntries].cRefs = 1;
884 pSet->aEntries[pSet->cEntries].iPage = iPage;
885 }
886 else
887 {
888 Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
889 int32_t i = pSet->cEntries;
890 while (i-- > 0)
891 if (pSet->aEntries[i].iPage)
892 {
893 pSet->aEntries[i].cRefs++;
894 break;
895 }
896 if (i < 0)
897 {
898 if (RT_UNLIKELY(pSet->cEntries >= RT_ELEMENTS(pSet->aEntries)))
899 pgmDynMapOptimizeAutoSet(pSet);
900 if (RT_LIKELY(pSet->cEntries < RT_ELEMENTS(pSet->aEntries)))
901 {
902 pSet->aEntries[pSet->cEntries].cRefs = 1;
903 pSet->aEntries[pSet->cEntries].iPage = iPage;
904 }
905 else
906 {
907 /* We're screwed. */
908 pgmR0DynMapReleasePage(g_pPGMR0DynMap, iPage, 1);
909
910 static uint32_t s_cBitched = 0;
911 if (++s_cBitched < 10)
912 LogRel(("PGMDynMapHCPage: set is full!\n"));
913 return VERR_PGM_DYNMAP_FULL_SET;
914 }
915 }
916 }
917
918 return VINF_SUCCESS;
919}
920
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette