PGM.cpp@ 27184

Last change on this file since 27184 was 27182, checked in by vboxsync, 15 years ago
Saved the state of ballooned memory and reinflate when loading the saved state.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 217.6 KB

Line
1	/* $Id: PGM.cpp 27182 2010-03-08 17:28:01Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor. (Mixing stuff here, not good?)
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/** @page pg_pgm PGM - The Page Manager and Monitor
24	*
25	* @see grp_pgm,
26	* @ref pg_pgm_pool,
27	* @ref pg_pgm_phys.
28	*
29	*
30	* @section sec_pgm_modes Paging Modes
31	*
32	* There are three memory contexts: Host Context (HC), Guest Context (GC)
33	* and intermediate context. When talking about paging HC can also be refered to
34	* as "host paging", and GC refered to as "shadow paging".
35	*
36	* We define three basic paging modes: 32-bit, PAE and AMD64. The host paging mode
37	* is defined by the host operating system. The mode used in the shadow paging mode
38	* depends on the host paging mode and what the mode the guest is currently in. The
39	* following relation between the two is defined:
40	*
41	* @verbatim
42	Host > 32-bit \| PAE \| AMD64 \|
43	Guest \| \| \| \|
44	==v================================
45	32-bit 32-bit PAE PAE
46	-------\|--------\|--------\|--------\|
47	PAE PAE PAE PAE
48	-------\|--------\|--------\|--------\|
49	AMD64 AMD64 AMD64 AMD64
50	-------\|--------\|--------\|--------\| @endverbatim
51	*
52	* All configuration except those in the diagonal (upper left) are expected to
53	* require special effort from the switcher (i.e. a bit slower).
54	*
55	*
56	*
57	*
58	* @section sec_pgm_shw The Shadow Memory Context
59	*
60	*
61	* [..]
62	*
63	* Because of guest context mappings requires PDPT and PML4 entries to allow
64	* writing on AMD64, the two upper levels will have fixed flags whatever the
65	* guest is thinking of using there. So, when shadowing the PD level we will
66	* calculate the effective flags of PD and all the higher levels. In legacy
67	* PAE mode this only applies to the PWT and PCD bits (the rest are
68	* ignored/reserved/MBZ). We will ignore those bits for the present.
69	*
70	*
71	*
72	* @section sec_pgm_int The Intermediate Memory Context
73	*
74	* The world switch goes thru an intermediate memory context which purpose it is
75	* to provide different mappings of the switcher code. All guest mappings are also
76	* present in this context.
77	*
78	* The switcher code is mapped at the same location as on the host, at an
79	* identity mapped location (physical equals virtual address), and at the
80	* hypervisor location. The identity mapped location is for when the world
81	* switches that involves disabling paging.
82	*
83	* PGM maintain page tables for 32-bit, PAE and AMD64 paging modes. This
84	* simplifies switching guest CPU mode and consistency at the cost of more
85	* code to do the work. All memory use for those page tables is located below
86	* 4GB (this includes page tables for guest context mappings).
87	*
88	*
89	* @subsection subsec_pgm_int_gc Guest Context Mappings
90	*
91	* During assignment and relocation of a guest context mapping the intermediate
92	* memory context is used to verify the new location.
93	*
94	* Guest context mappings are currently restricted to below 4GB, for reasons
95	* of simplicity. This may change when we implement AMD64 support.
96	*
97	*
98	*
99	*
100	* @section sec_pgm_misc Misc
101	*
102	* @subsection subsec_pgm_misc_diff Differences Between Legacy PAE and Long Mode PAE
103	*
104	* The differences between legacy PAE and long mode PAE are:
105	* -# PDPE bits 1, 2, 5 and 6 are defined differently. In leagcy mode they are
106	* all marked down as must-be-zero, while in long mode 1, 2 and 5 have the
107	* usual meanings while 6 is ignored (AMD). This means that upon switching to
108	* legacy PAE mode we'll have to clear these bits and when going to long mode
109	* they must be set. This applies to both intermediate and shadow contexts,
110	* however we don't need to do it for the intermediate one since we're
111	* executing with CR0.WP at that time.
112	* -# CR3 allows a 32-byte aligned address in legacy mode, while in long mode
113	* a page aligned one is required.
114	*
115	*
116	* @section sec_pgm_handlers Access Handlers
117	*
118	* Placeholder.
119	*
120	*
121	* @subsection sec_pgm_handlers_virt Virtual Access Handlers
122	*
123	* Placeholder.
124	*
125	*
126	* @subsection sec_pgm_handlers_virt Virtual Access Handlers
127	*
128	* We currently implement three types of virtual access handlers: ALL, WRITE
129	* and HYPERVISOR (WRITE). See PGMVIRTHANDLERTYPE for some more details.
130	*
131	* The HYPERVISOR access handlers is kept in a separate tree since it doesn't apply
132	* to physical pages (PGMTREES::HyperVirtHandlers) and only needs to be consulted in
133	* a special \#PF case. The ALL and WRITE are in the PGMTREES::VirtHandlers tree, the
134	* rest of this section is going to be about these handlers.
135	*
136	* We'll go thru the life cycle of a handler and try make sense of it all, don't know
137	* how successfull this is gonna be...
138	*
139	* 1. A handler is registered thru the PGMR3HandlerVirtualRegister and
140	* PGMHandlerVirtualRegisterEx APIs. We check for conflicting virtual handlers
141	* and create a new node that is inserted into the AVL tree (range key). Then
142	* a full PGM resync is flagged (clear pool, sync cr3, update virtual bit of PGMPAGE).
143	*
144	* 2. The following PGMSyncCR3/SyncCR3 operation will first make invoke HandlerVirtualUpdate.
145	*
146	* 2a. HandlerVirtualUpdate will will lookup all the pages covered by virtual handlers
147	* via the current guest CR3 and update the physical page -> virtual handler
148	* translation. Needless to say, this doesn't exactly scale very well. If any changes
149	* are detected, it will flag a virtual bit update just like we did on registration.
150	* PGMPHYS pages with changes will have their virtual handler state reset to NONE.
151	*
152	* 2b. The virtual bit update process will iterate all the pages covered by all the
153	* virtual handlers and update the PGMPAGE virtual handler state to the max of all
154	* virtual handlers on that page.
155	*
156	* 2c. Back in SyncCR3 we will now flush the entire shadow page cache to make sure
157	* we don't miss any alias mappings of the monitored pages.
158	*
159	* 2d. SyncCR3 will then proceed with syncing the CR3 table.
160	*
161	* 3. \#PF(np,read) on a page in the range. This will cause it to be synced
162	* read-only and resumed if it's a WRITE handler. If it's an ALL handler we
163	* will call the handlers like in the next step. If the physical mapping has
164	* changed we will - some time in the future - perform a handler callback
165	* (optional) and update the physical -> virtual handler cache.
166	*
167	* 4. \#PF(,write) on a page in the range. This will cause the handler to
168	* be invoked.
169	*
170	* 5. The guest invalidates the page and changes the physical backing or
171	* unmaps it. This should cause the invalidation callback to be invoked
172	* (it might not yet be 100% perfect). Exactly what happens next... is
173	* this where we mess up and end up out of sync for a while?
174	*
175	* 6. The handler is deregistered by the client via PGMHandlerVirtualDeregister.
176	* We will then set all PGMPAGEs in the physical -> virtual handler cache for
177	* this handler to NONE and trigger a full PGM resync (basically the same
178	* as int step 1). Which means 2 is executed again.
179	*
180	*
181	* @subsubsection sub_sec_pgm_handler_virt_todo TODOs
182	*
183	* There is a bunch of things that needs to be done to make the virtual handlers
184	* work 100% correctly and work more efficiently.
185	*
186	* The first bit hasn't been implemented yet because it's going to slow the
187	* whole mess down even more, and besides it seems to be working reliably for
188	* our current uses. OTOH, some of the optimizations might end up more or less
189	* implementing the missing bits, so we'll see.
190	*
191	* On the optimization side, the first thing to do is to try avoid unnecessary
192	* cache flushing. Then try team up with the shadowing code to track changes
193	* in mappings by means of access to them (shadow in), updates to shadows pages,
194	* invlpg, and shadow PT discarding (perhaps).
195	*
196	* Some idea that have popped up for optimization for current and new features:
197	* - bitmap indicating where there are virtual handlers installed.
198	* (4KB => 220 pages, page 212 => covers 32-bit address space 1:1!)
199	* - Further optimize this by min/max (needs min/max avl getters).
200	* - Shadow page table entry bit (if any left)?
201	*
202	*/
203
204
205	/** @page pg_pgm_phys PGM Physical Guest Memory Management
206	*
207	*
208	* Objectives:
209	* - Guest RAM over-commitment using memory ballooning,
210	* zero pages and general page sharing.
211	* - Moving or mirroring a VM onto a different physical machine.
212	*
213	*
214	* @subsection subsec_pgmPhys_Definitions Definitions
215	*
216	* Allocation chunk - A RTR0MemObjAllocPhysNC object and the tracking
217	* machinery assoicated with it.
218	*
219	*
220	*
221	*
222	* @subsection subsec_pgmPhys_AllocPage Allocating a page.
223	*
224	* Initially we map all guest memory to the (per VM) zero page, which
225	* means that none of the read functions will cause pages to be allocated.
226	*
227	* Exception, access bit in page tables that have been shared. This must
228	* be handled, but we must also make sure PGMGst*Modify doesn't make
229	* unnecessary modifications.
230	*
231	* Allocation points:
232	* - PGMPhysSimpleWriteGCPhys and PGMPhysWrite.
233	* - Replacing a zero page mapping at \#PF.
234	* - Replacing a shared page mapping at \#PF.
235	* - ROM registration (currently MMR3RomRegister).
236	* - VM restore (pgmR3Load).
237	*
238	* For the first three it would make sense to keep a few pages handy
239	* until we've reached the max memory commitment for the VM.
240	*
241	* For the ROM registration, we know exactly how many pages we need
242	* and will request these from ring-0. For restore, we will save
243	* the number of non-zero pages in the saved state and allocate
244	* them up front. This would allow the ring-0 component to refuse
245	* the request if the isn't sufficient memory available for VM use.
246	*
247	* Btw. for both ROM and restore allocations we won't be requiring
248	* zeroed pages as they are going to be filled instantly.
249	*
250	*
251	* @subsection subsec_pgmPhys_FreePage Freeing a page
252	*
253	* There are a few points where a page can be freed:
254	* - After being replaced by the zero page.
255	* - After being replaced by a shared page.
256	* - After being ballooned by the guest additions.
257	* - At reset.
258	* - At restore.
259	*
260	* When freeing one or more pages they will be returned to the ring-0
261	* component and replaced by the zero page.
262	*
263	* The reasoning for clearing out all the pages on reset is that it will
264	* return us to the exact same state as on power on, and may thereby help
265	* us reduce the memory load on the system. Further it might have a
266	* (temporary) positive influence on memory fragmentation (@see subsec_pgmPhys_Fragmentation).
267	*
268	* On restore, as mention under the allocation topic, pages should be
269	* freed / allocated depending on how many is actually required by the
270	* new VM state. The simplest approach is to do like on reset, and free
271	* all non-ROM pages and then allocate what we need.
272	*
273	* A measure to prevent some fragmentation, would be to let each allocation
274	* chunk have some affinity towards the VM having allocated the most pages
275	* from it. Also, try make sure to allocate from allocation chunks that
276	* are almost full. Admittedly, both these measures might work counter to
277	* our intentions and its probably not worth putting a lot of effort,
278	* cpu time or memory into this.
279	*
280	*
281	* @subsection subsec_pgmPhys_SharePage Sharing a page
282	*
283	* The basic idea is that there there will be a idle priority kernel
284	* thread walking the non-shared VM pages hashing them and looking for
285	* pages with the same checksum. If such pages are found, it will compare
286	* them byte-by-byte to see if they actually are identical. If found to be
287	* identical it will allocate a shared page, copy the content, check that
288	* the page didn't change while doing this, and finally request both the
289	* VMs to use the shared page instead. If the page is all zeros (special
290	* checksum and byte-by-byte check) it will request the VM that owns it
291	* to replace it with the zero page.
292	*
293	* To make this efficient, we will have to make sure not to try share a page
294	* that will change its contents soon. This part requires the most work.
295	* A simple idea would be to request the VM to write monitor the page for
296	* a while to make sure it isn't modified any time soon. Also, it may
297	* make sense to skip pages that are being write monitored since this
298	* information is readily available to the thread if it works on the
299	* per-VM guest memory structures (presently called PGMRAMRANGE).
300	*
301	*
302	* @subsection subsec_pgmPhys_Fragmentation Fragmentation Concerns and Counter Measures
303	*
304	* The pages are organized in allocation chunks in ring-0, this is a necessity
305	* if we wish to have an OS agnostic approach to this whole thing. (On Linux we
306	* could easily work on a page-by-page basis if we liked. Whether this is possible
307	* or efficient on NT I don't quite know.) Fragmentation within these chunks may
308	* become a problem as part of the idea here is that we wish to return memory to
309	* the host system.
310	*
311	* For instance, starting two VMs at the same time, they will both allocate the
312	* guest memory on-demand and if permitted their page allocations will be
313	* intermixed. Shut down one of the two VMs and it will be difficult to return
314	* any memory to the host system because the page allocation for the two VMs are
315	* mixed up in the same allocation chunks.
316	*
317	* To further complicate matters, when pages are freed because they have been
318	* ballooned or become shared/zero the whole idea is that the page is supposed
319	* to be reused by another VM or returned to the host system. This will cause
320	* allocation chunks to contain pages belonging to different VMs and prevent
321	* returning memory to the host when one of those VM shuts down.
322	*
323	* The only way to really deal with this problem is to move pages. This can
324	* either be done at VM shutdown and or by the idle priority worker thread
325	* that will be responsible for finding sharable/zero pages. The mechanisms
326	* involved for coercing a VM to move a page (or to do it for it) will be
327	* the same as when telling it to share/zero a page.
328	*
329	*
330	* @subsection subsec_pgmPhys_Tracking Tracking Structures And Their Cost
331	*
332	* There's a difficult balance between keeping the per-page tracking structures
333	* (global and guest page) easy to use and keeping them from eating too much
334	* memory. We have limited virtual memory resources available when operating in
335	* 32-bit kernel space (on 64-bit there'll it's quite a different story). The
336	* tracking structures will be attemted designed such that we can deal with up
337	* to 32GB of memory on a 32-bit system and essentially unlimited on 64-bit ones.
338	*
339	*
340	* @subsubsection subsubsec_pgmPhys_Tracking_Kernel Kernel Space
341	*
342	* @see pg_GMM
343	*
344	* @subsubsection subsubsec_pgmPhys_Tracking_PerVM Per-VM
345	*
346	* Fixed info is the physical address of the page (HCPhys) and the page id
347	* (described above). Theoretically we'll need 48(-12) bits for the HCPhys part.
348	* Today we've restricting ourselves to 40(-12) bits because this is the current
349	* restrictions of all AMD64 implementations (I think Barcelona will up this
350	* to 48(-12) bits, not that it really matters) and I needed the bits for
351	* tracking mappings of a page. 48-12 = 36. That leaves 28 bits, which means a
352	* decent range for the page id: 2^(28+12) = 1024TB.
353	*
354	* In additions to these, we'll have to keep maintaining the page flags as we
355	* currently do. Although it wouldn't harm to optimize these quite a bit, like
356	* for instance the ROM shouldn't depend on having a write handler installed
357	* in order for it to become read-only. A RO/RW bit should be considered so
358	* that the page syncing code doesn't have to mess about checking multiple
359	* flag combinations (ROM \|\| RW handler \|\| write monitored) in order to
360	* figure out how to setup a shadow PTE. But this of course, is second
361	* priority at present. Current this requires 12 bits, but could probably
362	* be optimized to ~8.
363	*
364	* Then there's the 24 bits used to track which shadow page tables are
365	* currently mapping a page for the purpose of speeding up physical
366	* access handlers, and thereby the page pool cache. More bit for this
367	* purpose wouldn't hurt IIRC.
368	*
369	* Then there is a new bit in which we need to record what kind of page
370	* this is, shared, zero, normal or write-monitored-normal. This'll
371	* require 2 bits. One bit might be needed for indicating whether a
372	* write monitored page has been written to. And yet another one or
373	* two for tracking migration status. 3-4 bits total then.
374	*
375	* Whatever is left will can be used to record the sharabilitiy of a
376	* page. The page checksum will not be stored in the per-VM table as
377	* the idle thread will not be permitted to do modifications to it.
378	* It will instead have to keep its own working set of potentially
379	* shareable pages and their check sums and stuff.
380	*
381	* For the present we'll keep the current packing of the
382	* PGMRAMRANGE::aHCPhys to keep the changes simple, only of course,
383	* we'll have to change it to a struct with a total of 128-bits at
384	* our disposal.
385	*
386	* The initial layout will be like this:
387	* @verbatim
388	RTHCPHYS HCPhys; The current stuff.
389	63:40 Current shadow PT tracking stuff.
390	39:12 The physical page frame number.
391	11:0 The current flags.
392	uint32_t u28PageId : 28; The page id.
393	uint32_t u2State : 2; The page state { zero, shared, normal, write monitored }.
394	uint32_t fWrittenTo : 1; Whether a write monitored page was written to.
395	uint32_t u1Reserved : 1; Reserved for later.
396	uint32_t u32Reserved; Reserved for later, mostly sharing stats.
397	@endverbatim
398	*
399	* The final layout will be something like this:
400	* @verbatim
401	RTHCPHYS HCPhys; The current stuff.
402	63:48 High page id (12+).
403	47:12 The physical page frame number.
404	11:0 Low page id.
405	uint32_t fReadOnly : 1; Whether it's readonly page (rom or monitored in some way).
406	uint32_t u3Type : 3; The page type {RESERVED, MMIO, MMIO2, ROM, shadowed ROM, RAM}.
407	uint32_t u2PhysMon : 2; Physical access handler type {none, read, write, all}.
408	uint32_t u2VirtMon : 2; Virtual access handler type {none, read, write, all}..
409	uint32_t u2State : 2; The page state { zero, shared, normal, write monitored }.
410	uint32_t fWrittenTo : 1; Whether a write monitored page was written to.
411	uint32_t u20Reserved : 20; Reserved for later, mostly sharing stats.
412	uint32_t u32Tracking; The shadow PT tracking stuff, roughly.
413	@endverbatim
414	*
415	* Cost wise, this means we'll double the cost for guest memory. There isn't anyway
416	* around that I'm afraid. It means that the cost of dealing out 32GB of memory
417	* to one or more VMs is: (32GB >> PAGE_SHIFT) * 16 bytes, or 128MBs. Or another
418	* example, the VM heap cost when assigning 1GB to a VM will be: 4MB.
419	*
420	* A couple of cost examples for the total cost per-VM + kernel.
421	* 32-bit Windows and 32-bit linux:
422	* 1GB guest ram, 256K pages: 4MB + 2MB(+) = 6MB
423	* 4GB guest ram, 1M pages: 16MB + 8MB(+) = 24MB
424	* 32GB guest ram, 8M pages: 128MB + 64MB(+) = 192MB
425	* 64-bit Windows and 64-bit linux:
426	* 1GB guest ram, 256K pages: 4MB + 3MB(+) = 7MB
427	* 4GB guest ram, 1M pages: 16MB + 12MB(+) = 28MB
428	* 32GB guest ram, 8M pages: 128MB + 96MB(+) = 224MB
429	*
430	* UPDATE - 2007-09-27:
431	* Will need a ballooned flag/state too because we cannot
432	* trust the guest 100% and reporting the same page as ballooned more
433	* than once will put the GMM off balance.
434	*
435	*
436	* @subsection subsec_pgmPhys_Serializing Serializing Access
437	*
438	* Initially, we'll try a simple scheme:
439	*
440	* - The per-VM RAM tracking structures (PGMRAMRANGE) is only modified
441	* by the EMT thread of that VM while in the pgm critsect.
442	* - Other threads in the VM process that needs to make reliable use of
443	* the per-VM RAM tracking structures will enter the critsect.
444	* - No process external thread or kernel thread will ever try enter
445	* the pgm critical section, as that just won't work.
446	* - The idle thread (and similar threads) doesn't not need 100% reliable
447	* data when performing it tasks as the EMT thread will be the one to
448	* do the actual changes later anyway. So, as long as it only accesses
449	* the main ram range, it can do so by somehow preventing the VM from
450	* being destroyed while it works on it...
451	*
452	* - The over-commitment management, including the allocating/freeing
453	* chunks, is serialized by a ring-0 mutex lock (a fast one since the
454	* more mundane mutex implementation is broken on Linux).
455	* - A separeate mutex is protecting the set of allocation chunks so
456	* that pages can be shared or/and freed up while some other VM is
457	* allocating more chunks. This mutex can be take from under the other
458	* one, but not the otherway around.
459	*
460	*
461	* @subsection subsec_pgmPhys_Request VM Request interface
462	*
463	* When in ring-0 it will become necessary to send requests to a VM so it can
464	* for instance move a page while defragmenting during VM destroy. The idle
465	* thread will make use of this interface to request VMs to setup shared
466	* pages and to perform write monitoring of pages.
467	*
468	* I would propose an interface similar to the current VMReq interface, similar
469	* in that it doesn't require locking and that the one sending the request may
470	* wait for completion if it wishes to. This shouldn't be very difficult to
471	* realize.
472	*
473	* The requests themselves are also pretty simple. They are basically:
474	* -# Check that some precondition is still true.
475	* -# Do the update.
476	* -# Update all shadow page tables involved with the page.
477	*
478	* The 3rd step is identical to what we're already doing when updating a
479	* physical handler, see pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs.
480	*
481	*
482	*
483	* @section sec_pgmPhys_MappingCaches Mapping Caches
484	*
485	* In order to be able to map in and out memory and to be able to support
486	* guest with more RAM than we've got virtual address space, we'll employing
487	* a mapping cache. There is already a tiny one for GC (see PGMGCDynMapGCPageEx)
488	* and we'll create a similar one for ring-0 unless we decide to setup a dedicate
489	* memory context for the HWACCM execution.
490	*
491	*
492	* @subsection subsec_pgmPhys_MappingCaches_R3 Ring-3
493	*
494	* We've considered implementing the ring-3 mapping cache page based but found
495	* that this was bother some when one had to take into account TLBs+SMP and
496	* portability (missing the necessary APIs on several platforms). There were
497	* also some performance concerns with this approach which hadn't quite been
498	* worked out.
499	*
500	* Instead, we'll be mapping allocation chunks into the VM process. This simplifies
501	* matters greatly quite a bit since we don't need to invent any new ring-0 stuff,
502	* only some minor RTR0MEMOBJ mapping stuff. The main concern here is that mapping
503	* compared to the previous idea is that mapping or unmapping a 1MB chunk is more
504	* costly than a single page, although how much more costly is uncertain. We'll
505	* try address this by using a very big cache, preferably bigger than the actual
506	* VM RAM size if possible. The current VM RAM sizes should give some idea for
507	* 32-bit boxes, while on 64-bit we can probably get away with employing an
508	* unlimited cache.
509	*
510	* The cache have to parts, as already indicated, the ring-3 side and the
511	* ring-0 side.
512	*
513	* The ring-0 will be tied to the page allocator since it will operate on the
514	* memory objects it contains. It will therefore require the first ring-0 mutex
515	* discussed in @ref subsec_pgmPhys_Serializing. We
516	* some double house keeping wrt to who has mapped what I think, since both
517	* VMMR0.r0 and RTR0MemObj will keep track of mapping relataions
518	*
519	* The ring-3 part will be protected by the pgm critsect. For simplicity, we'll
520	* require anyone that desires to do changes to the mapping cache to do that
521	* from within this critsect. Alternatively, we could employ a separate critsect
522	* for serializing changes to the mapping cache as this would reduce potential
523	* contention with other threads accessing mappings unrelated to the changes
524	* that are in process. We can see about this later, contention will show
525	* up in the statistics anyway, so it'll be simple to tell.
526	*
527	* The organization of the ring-3 part will be very much like how the allocation
528	* chunks are organized in ring-0, that is in an AVL tree by chunk id. To avoid
529	* having to walk the tree all the time, we'll have a couple of lookaside entries
530	* like in we do for I/O ports and MMIO in IOM.
531	*
532	* The simplified flow of a PGMPhysRead/Write function:
533	* -# Enter the PGM critsect.
534	* -# Lookup GCPhys in the ram ranges and get the Page ID.
535	* -# Calc the Allocation Chunk ID from the Page ID.
536	* -# Check the lookaside entries and then the AVL tree for the Chunk ID.
537	* If not found in cache:
538	* -# Call ring-0 and request it to be mapped and supply
539	* a chunk to be unmapped if the cache is maxed out already.
540	* -# Insert the new mapping into the AVL tree (id + R3 address).
541	* -# Update the relevant lookaside entry and return the mapping address.
542	* -# Do the read/write according to monitoring flags and everything.
543	* -# Leave the critsect.
544	*
545	*
546	* @section sec_pgmPhys_Fallback Fallback
547	*
548	* Current all the "second tier" hosts will not support the RTR0MemObjAllocPhysNC
549	* API and thus require a fallback.
550	*
551	* So, when RTR0MemObjAllocPhysNC returns VERR_NOT_SUPPORTED the page allocator
552	* will return to the ring-3 caller (and later ring-0) and asking it to seed
553	* the page allocator with some fresh pages (VERR_GMM_SEED_ME). Ring-3 will
554	* then perform an SUPR3PageAlloc(cbChunk >> PAGE_SHIFT) call and make a
555	* "SeededAllocPages" call to ring-0.
556	*
557	* The first time ring-0 sees the VERR_NOT_SUPPORTED failure it will disable
558	* all page sharing (zero page detection will continue). It will also force
559	* all allocations to come from the VM which seeded the page. Both these
560	* measures are taken to make sure that there will never be any need for
561	* mapping anything into ring-3 - everything will be mapped already.
562	*
563	* Whether we'll continue to use the current MM locked memory management
564	* for this I don't quite know (I'd prefer not to and just ditch that all
565	* togther), we'll see what's simplest to do.
566	*
567	*
568	*
569	* @section sec_pgmPhys_Changes Changes
570	*
571	* Breakdown of the changes involved?
572	*/
573
574	/*******************************************************************************
575	* Header Files *
576	*******************************************************************************/
577	#define LOG_GROUP LOG_GROUP_PGM
578	#include <VBox/dbgf.h>
579	#include <VBox/pgm.h>
580	#include <VBox/cpum.h>
581	#include <VBox/iom.h>
582	#include <VBox/sup.h>
583	#include <VBox/mm.h>
584	#include <VBox/em.h>
585	#include <VBox/stam.h>
586	#include <VBox/rem.h>
587	#include <VBox/selm.h>
588	#include <VBox/ssm.h>
589	#include <VBox/hwaccm.h>
590	#include "PGMInternal.h"
591	#include <VBox/vm.h>
592	#include "PGMInline.h"
593
594	#include <VBox/dbg.h>
595	#include <VBox/param.h>
596	#include <VBox/err.h>
597
598	#include <iprt/asm.h>
599	#include <iprt/assert.h>
600	#include <iprt/env.h>
601	#include <iprt/mem.h>
602	#include <iprt/file.h>
603	#include <iprt/string.h>
604	#include <iprt/thread.h>
605
606
607	/*******************************************************************************
608	* Defined Constants And Macros *
609	*******************************************************************************/
610	/** Saved state data unit version for 2.5.x and later. */
611	#define PGM_SAVED_STATE_VERSION 9
612	/** Saved state data unit version for 2.2.2 and later. */
613	#define PGM_SAVED_STATE_VERSION_2_2_2 8
614	/** Saved state data unit version for 2.2.0. */
615	#define PGM_SAVED_STATE_VERSION_RR_DESC 7
616	/** Saved state data unit version. */
617	#define PGM_SAVED_STATE_VERSION_OLD_PHYS_CODE 6
618
619
620	/*******************************************************************************
621	* Internal Functions *
622	*******************************************************************************/
623	static int pgmR3InitPaging(PVM pVM);
624	static void pgmR3InitStats(PVM pVM);
625	static DECLCALLBACK(void) pgmR3PhysInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
626	static DECLCALLBACK(void) pgmR3InfoMode(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
627	static DECLCALLBACK(void) pgmR3InfoCr3(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
628	static DECLCALLBACK(int) pgmR3RelocatePhysHandler(PAVLROGCPHYSNODECORE pNode, void *pvUser);
629	static DECLCALLBACK(int) pgmR3RelocateVirtHandler(PAVLROGCPTRNODECORE pNode, void *pvUser);
630	static DECLCALLBACK(int) pgmR3RelocateHyperVirtHandler(PAVLROGCPTRNODECORE pNode, void *pvUser);
631	#ifdef VBOX_STRICT
632	static DECLCALLBACK(void) pgmR3ResetNoMorePhysWritesFlag(PVM pVM, VMSTATE enmState, VMSTATE enmOldState, void *pvUser);
633	#endif
634	static int pgmR3ModeDataInit(PVM pVM, bool fResolveGCAndR0);
635	static void pgmR3ModeDataSwitch(PVM pVM, PVMCPU pVCpu, PGMMODE enmShw, PGMMODE enmGst);
636	static PGMMODE pgmR3CalcShadowMode(PVM pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode, VMMSWITCHER *penmSwitcher);
637
638	#ifdef VBOX_WITH_DEBUGGER
639	/** @todo Convert the first two commands to 'info' items. */
640	static DECLCALLBACK(int) pgmR3CmdRam(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
641	static DECLCALLBACK(int) pgmR3CmdError(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
642	static DECLCALLBACK(int) pgmR3CmdSync(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
643	static DECLCALLBACK(int) pgmR3CmdSyncAlways(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
644	# ifdef VBOX_STRICT
645	static DECLCALLBACK(int) pgmR3CmdAssertCR3(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
646	# endif
647	static DECLCALLBACK(int) pgmR3CmdPhysToFile(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
648	#endif
649
650
651	/*******************************************************************************
652	* Global Variables *
653	*******************************************************************************/
654	#ifdef VBOX_WITH_DEBUGGER
655	/** Argument descriptors for '.pgmerror' and '.pgmerroroff'. */
656	static const DBGCVARDESC g_aPgmErrorArgs[] =
657	{
658	/* cTimesMin, cTimesMax, enmCategory, fFlags, pszName, pszDescription */
659	{ 0, 1, DBGCVAR_CAT_STRING, 0, "where", "Error injection location." },
660	};
661
662	static const DBGCVARDESC g_aPgmPhysToFileArgs[] =
663	{
664	/* cTimesMin, cTimesMax, enmCategory, fFlags, pszName, pszDescription */
665	{ 1, 1, DBGCVAR_CAT_STRING, 0, "file", "The file name." },
666	{ 0, 1, DBGCVAR_CAT_STRING, 0, "nozero", "If present, zero pages are skipped." },
667	};
668
669	/** Command descriptors. */
670	static const DBGCCMD g_aCmds[] =
671	{
672	/* pszCmd, cArgsMin, cArgsMax, paArgDesc, cArgDescs, pResultDesc, fFlags, pfnHandler pszSyntax, ....pszDescription */
673	{ "pgmram", 0, 0, NULL, 0, NULL, 0, pgmR3CmdRam, "", "Display the ram ranges." },
674	{ "pgmsync", 0, 0, NULL, 0, NULL, 0, pgmR3CmdSync, "", "Sync the CR3 page." },
675	{ "pgmerror", 0, 1, &g_aPgmErrorArgs[0], 1, NULL, 0, pgmR3CmdError, "", "Enables inject runtime of errors into parts of PGM." },
676	{ "pgmerroroff", 0, 1, &g_aPgmErrorArgs[0], 1, NULL, 0, pgmR3CmdError, "", "Disables inject runtime errors into parts of PGM." },
677	#ifdef VBOX_STRICT
678	{ "pgmassertcr3", 0, 0, NULL, 0, NULL, 0, pgmR3CmdAssertCR3, "", "Check the shadow CR3 mapping." },
679	#endif
680	{ "pgmsyncalways", 0, 0, NULL, 0, NULL, 0, pgmR3CmdSyncAlways, "", "Toggle permanent CR3 syncing." },
681	{ "pgmphystofile", 1, 2, &g_aPgmPhysToFileArgs[0], 2, NULL, 0, pgmR3CmdPhysToFile, "", "Save the physical memory to file." },
682	};
683	#endif
684
685
686
687
688	/*
689	* Shadow - 32-bit mode
690	*/
691	#define PGM_SHW_TYPE PGM_TYPE_32BIT
692	#define PGM_SHW_NAME(name) PGM_SHW_NAME_32BIT(name)
693	#define PGM_SHW_NAME_RC_STR(name) PGM_SHW_NAME_RC_32BIT_STR(name)
694	#define PGM_SHW_NAME_R0_STR(name) PGM_SHW_NAME_R0_32BIT_STR(name)
695	#include "PGMShw.h"
696
697	/* Guest - real mode */
698	#define PGM_GST_TYPE PGM_TYPE_REAL
699	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
700	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_REAL_STR(name)
701	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_REAL_STR(name)
702	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_REAL(name)
703	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_32BIT_REAL_STR(name)
704	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_32BIT_REAL_STR(name)
705	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
706	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
707	#include "PGMBth.h"
708	#include "PGMGstDefs.h"
709	#include "PGMGst.h"
710	#undef BTH_PGMPOOLKIND_PT_FOR_PT
711	#undef BTH_PGMPOOLKIND_ROOT
712	#undef PGM_BTH_NAME
713	#undef PGM_BTH_NAME_RC_STR
714	#undef PGM_BTH_NAME_R0_STR
715	#undef PGM_GST_TYPE
716	#undef PGM_GST_NAME
717	#undef PGM_GST_NAME_RC_STR
718	#undef PGM_GST_NAME_R0_STR
719
720	/* Guest - protected mode */
721	#define PGM_GST_TYPE PGM_TYPE_PROT
722	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
723	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_PROT_STR(name)
724	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_PROT_STR(name)
725	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name)
726	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_32BIT_PROT_STR(name)
727	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_32BIT_PROT_STR(name)
728	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
729	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
730	#include "PGMBth.h"
731	#include "PGMGstDefs.h"
732	#include "PGMGst.h"
733	#undef BTH_PGMPOOLKIND_PT_FOR_PT
734	#undef BTH_PGMPOOLKIND_ROOT
735	#undef PGM_BTH_NAME
736	#undef PGM_BTH_NAME_RC_STR
737	#undef PGM_BTH_NAME_R0_STR
738	#undef PGM_GST_TYPE
739	#undef PGM_GST_NAME
740	#undef PGM_GST_NAME_RC_STR
741	#undef PGM_GST_NAME_R0_STR
742
743	/* Guest - 32-bit mode */
744	#define PGM_GST_TYPE PGM_TYPE_32BIT
745	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
746	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_32BIT_STR(name)
747	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_32BIT_STR(name)
748	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_32BIT(name)
749	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_32BIT_32BIT_STR(name)
750	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_32BIT_32BIT_STR(name)
751	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
752	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
753	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD
754	#include "PGMBth.h"
755	#include "PGMGstDefs.h"
756	#include "PGMGst.h"
757	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
758	#undef BTH_PGMPOOLKIND_PT_FOR_PT
759	#undef BTH_PGMPOOLKIND_ROOT
760	#undef PGM_BTH_NAME
761	#undef PGM_BTH_NAME_RC_STR
762	#undef PGM_BTH_NAME_R0_STR
763	#undef PGM_GST_TYPE
764	#undef PGM_GST_NAME
765	#undef PGM_GST_NAME_RC_STR
766	#undef PGM_GST_NAME_R0_STR
767
768	#undef PGM_SHW_TYPE
769	#undef PGM_SHW_NAME
770	#undef PGM_SHW_NAME_RC_STR
771	#undef PGM_SHW_NAME_R0_STR
772
773
774	/*
775	* Shadow - PAE mode
776	*/
777	#define PGM_SHW_TYPE PGM_TYPE_PAE
778	#define PGM_SHW_NAME(name) PGM_SHW_NAME_PAE(name)
779	#define PGM_SHW_NAME_RC_STR(name) PGM_SHW_NAME_RC_PAE_STR(name)
780	#define PGM_SHW_NAME_R0_STR(name) PGM_SHW_NAME_R0_PAE_STR(name)
781	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
782	#include "PGMShw.h"
783
784	/* Guest - real mode */
785	#define PGM_GST_TYPE PGM_TYPE_REAL
786	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
787	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_REAL_STR(name)
788	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_REAL_STR(name)
789	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
790	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_PAE_REAL_STR(name)
791	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_PAE_REAL_STR(name)
792	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
793	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
794	#include "PGMGstDefs.h"
795	#include "PGMBth.h"
796	#undef BTH_PGMPOOLKIND_PT_FOR_PT
797	#undef BTH_PGMPOOLKIND_ROOT
798	#undef PGM_BTH_NAME
799	#undef PGM_BTH_NAME_RC_STR
800	#undef PGM_BTH_NAME_R0_STR
801	#undef PGM_GST_TYPE
802	#undef PGM_GST_NAME
803	#undef PGM_GST_NAME_RC_STR
804	#undef PGM_GST_NAME_R0_STR
805
806	/* Guest - protected mode */
807	#define PGM_GST_TYPE PGM_TYPE_PROT
808	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
809	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_PROT_STR(name)
810	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_PROT_STR(name)
811	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name)
812	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_PAE_PROT_STR(name)
813	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_PAE_PROT_STR(name)
814	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
815	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
816	#include "PGMGstDefs.h"
817	#include "PGMBth.h"
818	#undef BTH_PGMPOOLKIND_PT_FOR_PT
819	#undef BTH_PGMPOOLKIND_ROOT
820	#undef PGM_BTH_NAME
821	#undef PGM_BTH_NAME_RC_STR
822	#undef PGM_BTH_NAME_R0_STR
823	#undef PGM_GST_TYPE
824	#undef PGM_GST_NAME
825	#undef PGM_GST_NAME_RC_STR
826	#undef PGM_GST_NAME_R0_STR
827
828	/* Guest - 32-bit mode */
829	#define PGM_GST_TYPE PGM_TYPE_32BIT
830	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
831	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_32BIT_STR(name)
832	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_32BIT_STR(name)
833	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_32BIT(name)
834	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_PAE_32BIT_STR(name)
835	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_PAE_32BIT_STR(name)
836	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
837	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
838	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_FOR_32BIT
839	#include "PGMGstDefs.h"
840	#include "PGMBth.h"
841	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
842	#undef BTH_PGMPOOLKIND_PT_FOR_PT
843	#undef BTH_PGMPOOLKIND_ROOT
844	#undef PGM_BTH_NAME
845	#undef PGM_BTH_NAME_RC_STR
846	#undef PGM_BTH_NAME_R0_STR
847	#undef PGM_GST_TYPE
848	#undef PGM_GST_NAME
849	#undef PGM_GST_NAME_RC_STR
850	#undef PGM_GST_NAME_R0_STR
851
852	/* Guest - PAE mode */
853	#define PGM_GST_TYPE PGM_TYPE_PAE
854	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
855	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_PAE_STR(name)
856	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_PAE_STR(name)
857	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PAE(name)
858	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_PAE_PAE_STR(name)
859	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_PAE_PAE_STR(name)
860	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
861	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
862	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT
863	#include "PGMBth.h"
864	#include "PGMGstDefs.h"
865	#include "PGMGst.h"
866	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
867	#undef BTH_PGMPOOLKIND_PT_FOR_PT
868	#undef BTH_PGMPOOLKIND_ROOT
869	#undef PGM_BTH_NAME
870	#undef PGM_BTH_NAME_RC_STR
871	#undef PGM_BTH_NAME_R0_STR
872	#undef PGM_GST_TYPE
873	#undef PGM_GST_NAME
874	#undef PGM_GST_NAME_RC_STR
875	#undef PGM_GST_NAME_R0_STR
876
877	#undef PGM_SHW_TYPE
878	#undef PGM_SHW_NAME
879	#undef PGM_SHW_NAME_RC_STR
880	#undef PGM_SHW_NAME_R0_STR
881
882
883	/*
884	* Shadow - AMD64 mode
885	*/
886	#define PGM_SHW_TYPE PGM_TYPE_AMD64
887	#define PGM_SHW_NAME(name) PGM_SHW_NAME_AMD64(name)
888	#define PGM_SHW_NAME_RC_STR(name) PGM_SHW_NAME_RC_AMD64_STR(name)
889	#define PGM_SHW_NAME_R0_STR(name) PGM_SHW_NAME_R0_AMD64_STR(name)
890	#include "PGMShw.h"
891
892	#ifdef VBOX_WITH_64_BITS_GUESTS
893	/* Guest - AMD64 mode */
894	# define PGM_GST_TYPE PGM_TYPE_AMD64
895	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
896	# define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_AMD64_STR(name)
897	# define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_AMD64_STR(name)
898	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_AMD64(name)
899	# define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_AMD64_AMD64_STR(name)
900	# define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_AMD64_AMD64_STR(name)
901	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
902	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
903	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_64BIT_PML4
904	# include "PGMBth.h"
905	# include "PGMGstDefs.h"
906	# include "PGMGst.h"
907	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
908	# undef BTH_PGMPOOLKIND_PT_FOR_PT
909	# undef BTH_PGMPOOLKIND_ROOT
910	# undef PGM_BTH_NAME
911	# undef PGM_BTH_NAME_RC_STR
912	# undef PGM_BTH_NAME_R0_STR
913	# undef PGM_GST_TYPE
914	# undef PGM_GST_NAME
915	# undef PGM_GST_NAME_RC_STR
916	# undef PGM_GST_NAME_R0_STR
917	#endif /* VBOX_WITH_64_BITS_GUESTS */
918
919	#undef PGM_SHW_TYPE
920	#undef PGM_SHW_NAME
921	#undef PGM_SHW_NAME_RC_STR
922	#undef PGM_SHW_NAME_R0_STR
923
924
925	/*
926	* Shadow - Nested paging mode
927	*/
928	#define PGM_SHW_TYPE PGM_TYPE_NESTED
929	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED(name)
930	#define PGM_SHW_NAME_RC_STR(name) PGM_SHW_NAME_RC_NESTED_STR(name)
931	#define PGM_SHW_NAME_R0_STR(name) PGM_SHW_NAME_R0_NESTED_STR(name)
932	#include "PGMShw.h"
933
934	/* Guest - real mode */
935	#define PGM_GST_TYPE PGM_TYPE_REAL
936	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
937	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_REAL_STR(name)
938	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_REAL_STR(name)
939	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_REAL(name)
940	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_NESTED_REAL_STR(name)
941	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_NESTED_REAL_STR(name)
942	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
943	#include "PGMGstDefs.h"
944	#include "PGMBth.h"
945	#undef BTH_PGMPOOLKIND_PT_FOR_PT
946	#undef PGM_BTH_NAME
947	#undef PGM_BTH_NAME_RC_STR
948	#undef PGM_BTH_NAME_R0_STR
949	#undef PGM_GST_TYPE
950	#undef PGM_GST_NAME
951	#undef PGM_GST_NAME_RC_STR
952	#undef PGM_GST_NAME_R0_STR
953
954	/* Guest - protected mode */
955	#define PGM_GST_TYPE PGM_TYPE_PROT
956	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
957	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_PROT_STR(name)
958	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_PROT_STR(name)
959	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PROT(name)
960	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_NESTED_PROT_STR(name)
961	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_NESTED_PROT_STR(name)
962	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
963	#include "PGMGstDefs.h"
964	#include "PGMBth.h"
965	#undef BTH_PGMPOOLKIND_PT_FOR_PT
966	#undef PGM_BTH_NAME
967	#undef PGM_BTH_NAME_RC_STR
968	#undef PGM_BTH_NAME_R0_STR
969	#undef PGM_GST_TYPE
970	#undef PGM_GST_NAME
971	#undef PGM_GST_NAME_RC_STR
972	#undef PGM_GST_NAME_R0_STR
973
974	/* Guest - 32-bit mode */
975	#define PGM_GST_TYPE PGM_TYPE_32BIT
976	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
977	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_32BIT_STR(name)
978	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_32BIT_STR(name)
979	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT(name)
980	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_NESTED_32BIT_STR(name)
981	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_NESTED_32BIT_STR(name)
982	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
983	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
984	#include "PGMGstDefs.h"
985	#include "PGMBth.h"
986	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
987	#undef BTH_PGMPOOLKIND_PT_FOR_PT
988	#undef PGM_BTH_NAME
989	#undef PGM_BTH_NAME_RC_STR
990	#undef PGM_BTH_NAME_R0_STR
991	#undef PGM_GST_TYPE
992	#undef PGM_GST_NAME
993	#undef PGM_GST_NAME_RC_STR
994	#undef PGM_GST_NAME_R0_STR
995
996	/* Guest - PAE mode */
997	#define PGM_GST_TYPE PGM_TYPE_PAE
998	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
999	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_PAE_STR(name)
1000	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_PAE_STR(name)
1001	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE(name)
1002	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_NESTED_PAE_STR(name)
1003	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_NESTED_PAE_STR(name)
1004	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1005	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
1006	#include "PGMGstDefs.h"
1007	#include "PGMBth.h"
1008	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
1009	#undef BTH_PGMPOOLKIND_PT_FOR_PT
1010	#undef PGM_BTH_NAME
1011	#undef PGM_BTH_NAME_RC_STR
1012	#undef PGM_BTH_NAME_R0_STR
1013	#undef PGM_GST_TYPE
1014	#undef PGM_GST_NAME
1015	#undef PGM_GST_NAME_RC_STR
1016	#undef PGM_GST_NAME_R0_STR
1017
1018	#ifdef VBOX_WITH_64_BITS_GUESTS
1019	/* Guest - AMD64 mode */
1020	# define PGM_GST_TYPE PGM_TYPE_AMD64
1021	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
1022	# define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_AMD64_STR(name)
1023	# define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_AMD64_STR(name)
1024	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64(name)
1025	# define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_NESTED_AMD64_STR(name)
1026	# define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_NESTED_AMD64_STR(name)
1027	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1028	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
1029	# include "PGMGstDefs.h"
1030	# include "PGMBth.h"
1031	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
1032	# undef BTH_PGMPOOLKIND_PT_FOR_PT
1033	# undef PGM_BTH_NAME
1034	# undef PGM_BTH_NAME_RC_STR
1035	# undef PGM_BTH_NAME_R0_STR
1036	# undef PGM_GST_TYPE
1037	# undef PGM_GST_NAME
1038	# undef PGM_GST_NAME_RC_STR
1039	# undef PGM_GST_NAME_R0_STR
1040	#endif /* VBOX_WITH_64_BITS_GUESTS */
1041
1042	#undef PGM_SHW_TYPE
1043	#undef PGM_SHW_NAME
1044	#undef PGM_SHW_NAME_RC_STR
1045	#undef PGM_SHW_NAME_R0_STR
1046
1047
1048	/*
1049	* Shadow - EPT
1050	*/
1051	#define PGM_SHW_TYPE PGM_TYPE_EPT
1052	#define PGM_SHW_NAME(name) PGM_SHW_NAME_EPT(name)
1053	#define PGM_SHW_NAME_RC_STR(name) PGM_SHW_NAME_RC_EPT_STR(name)
1054	#define PGM_SHW_NAME_R0_STR(name) PGM_SHW_NAME_R0_EPT_STR(name)
1055	#include "PGMShw.h"
1056
1057	/* Guest - real mode */
1058	#define PGM_GST_TYPE PGM_TYPE_REAL
1059	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
1060	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_REAL_STR(name)
1061	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_REAL_STR(name)
1062	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_REAL(name)
1063	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_EPT_REAL_STR(name)
1064	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_EPT_REAL_STR(name)
1065	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
1066	#include "PGMGstDefs.h"
1067	#include "PGMBth.h"
1068	#undef BTH_PGMPOOLKIND_PT_FOR_PT
1069	#undef PGM_BTH_NAME
1070	#undef PGM_BTH_NAME_RC_STR
1071	#undef PGM_BTH_NAME_R0_STR
1072	#undef PGM_GST_TYPE
1073	#undef PGM_GST_NAME
1074	#undef PGM_GST_NAME_RC_STR
1075	#undef PGM_GST_NAME_R0_STR
1076
1077	/* Guest - protected mode */
1078	#define PGM_GST_TYPE PGM_TYPE_PROT
1079	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
1080	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_PROT_STR(name)
1081	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_PROT_STR(name)
1082	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name)
1083	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_EPT_PROT_STR(name)
1084	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_EPT_PROT_STR(name)
1085	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
1086	#include "PGMGstDefs.h"
1087	#include "PGMBth.h"
1088	#undef BTH_PGMPOOLKIND_PT_FOR_PT
1089	#undef PGM_BTH_NAME
1090	#undef PGM_BTH_NAME_RC_STR
1091	#undef PGM_BTH_NAME_R0_STR
1092	#undef PGM_GST_TYPE
1093	#undef PGM_GST_NAME
1094	#undef PGM_GST_NAME_RC_STR
1095	#undef PGM_GST_NAME_R0_STR
1096
1097	/* Guest - 32-bit mode */
1098	#define PGM_GST_TYPE PGM_TYPE_32BIT
1099	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
1100	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_32BIT_STR(name)
1101	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_32BIT_STR(name)
1102	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_32BIT(name)
1103	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_EPT_32BIT_STR(name)
1104	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_EPT_32BIT_STR(name)
1105	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
1106	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
1107	#include "PGMGstDefs.h"
1108	#include "PGMBth.h"
1109	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
1110	#undef BTH_PGMPOOLKIND_PT_FOR_PT
1111	#undef PGM_BTH_NAME
1112	#undef PGM_BTH_NAME_RC_STR
1113	#undef PGM_BTH_NAME_R0_STR
1114	#undef PGM_GST_TYPE
1115	#undef PGM_GST_NAME
1116	#undef PGM_GST_NAME_RC_STR
1117	#undef PGM_GST_NAME_R0_STR
1118
1119	/* Guest - PAE mode */
1120	#define PGM_GST_TYPE PGM_TYPE_PAE
1121	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
1122	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_PAE_STR(name)
1123	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_PAE_STR(name)
1124	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PAE(name)
1125	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_EPT_PAE_STR(name)
1126	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_EPT_PAE_STR(name)
1127	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1128	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
1129	#include "PGMGstDefs.h"
1130	#include "PGMBth.h"
1131	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
1132	#undef BTH_PGMPOOLKIND_PT_FOR_PT
1133	#undef PGM_BTH_NAME
1134	#undef PGM_BTH_NAME_RC_STR
1135	#undef PGM_BTH_NAME_R0_STR
1136	#undef PGM_GST_TYPE
1137	#undef PGM_GST_NAME
1138	#undef PGM_GST_NAME_RC_STR
1139	#undef PGM_GST_NAME_R0_STR
1140
1141	#ifdef VBOX_WITH_64_BITS_GUESTS
1142	/* Guest - AMD64 mode */
1143	# define PGM_GST_TYPE PGM_TYPE_AMD64
1144	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
1145	# define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_AMD64_STR(name)
1146	# define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_AMD64_STR(name)
1147	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_AMD64(name)
1148	# define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_EPT_AMD64_STR(name)
1149	# define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_EPT_AMD64_STR(name)
1150	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1151	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
1152	# include "PGMGstDefs.h"
1153	# include "PGMBth.h"
1154	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
1155	# undef BTH_PGMPOOLKIND_PT_FOR_PT
1156	# undef PGM_BTH_NAME
1157	# undef PGM_BTH_NAME_RC_STR
1158	# undef PGM_BTH_NAME_R0_STR
1159	# undef PGM_GST_TYPE
1160	# undef PGM_GST_NAME
1161	# undef PGM_GST_NAME_RC_STR
1162	# undef PGM_GST_NAME_R0_STR
1163	#endif /* VBOX_WITH_64_BITS_GUESTS */
1164
1165	#undef PGM_SHW_TYPE
1166	#undef PGM_SHW_NAME
1167	#undef PGM_SHW_NAME_RC_STR
1168	#undef PGM_SHW_NAME_R0_STR
1169
1170
1171
1172	/**
1173	* Initiates the paging of VM.
1174	*
1175	* @returns VBox status code.
1176	* @param pVM Pointer to VM structure.
1177	*/
1178	VMMR3DECL(int) PGMR3Init(PVM pVM)
1179	{
1180	LogFlow(("PGMR3Init:\n"));
1181	PCFGMNODE pCfgPGM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/PGM");
1182	int rc;
1183
1184	/*
1185	* Assert alignment and sizes.
1186	*/
1187	AssertCompile(sizeof(pVM->pgm.s) <= sizeof(pVM->pgm.padding));
1188	AssertCompile(sizeof(pVM->aCpus[0].pgm.s) <= sizeof(pVM->aCpus[0].pgm.padding));
1189	AssertCompileMemberAlignment(PGM, CritSect, sizeof(uintptr_t));
1190
1191	/*
1192	* Init the structure.
1193	*/
1194	#ifdef PGM_WITHOUT_MAPPINGS
1195	pVM->pgm.s.fMappingsDisabled = true;
1196	#endif
1197	pVM->pgm.s.offVM = RT_OFFSETOF(VM, pgm.s);
1198	pVM->pgm.s.offVCpuPGM = RT_OFFSETOF(VMCPU, pgm.s);
1199
1200	/* Init the per-CPU part. */
1201	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1202	{
1203	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1204	PPGMCPU pPGM = &pVCpu->pgm.s;
1205
1206	pPGM->offVM = (uintptr_t)&pVCpu->pgm.s - (uintptr_t)pVM;
1207	pPGM->offVCpu = RT_OFFSETOF(VMCPU, pgm.s);
1208	pPGM->offPGM = (uintptr_t)&pVCpu->pgm.s - (uintptr_t)&pVM->pgm.s;
1209
1210	pPGM->enmShadowMode = PGMMODE_INVALID;
1211	pPGM->enmGuestMode = PGMMODE_INVALID;
1212
1213	pPGM->GCPhysCR3 = NIL_RTGCPHYS;
1214
1215	pPGM->pGstPaePdptR3 = NULL;
1216	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
1217	pPGM->pGstPaePdptR0 = NIL_RTR0PTR;
1218	#endif
1219	pPGM->pGstPaePdptRC = NIL_RTRCPTR;
1220	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.apGstPaePDsR3); i++)
1221	{
1222	pPGM->apGstPaePDsR3[i] = NULL;
1223	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
1224	pPGM->apGstPaePDsR0[i] = NIL_RTR0PTR;
1225	#endif
1226	pPGM->apGstPaePDsRC[i] = NIL_RTRCPTR;
1227	pPGM->aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
1228	pPGM->aGCPhysGstPaePDsMonitored[i] = NIL_RTGCPHYS;
1229	}
1230
1231	pPGM->fA20Enabled = true;
1232	}
1233
1234	pVM->pgm.s.enmHostMode = SUPPAGINGMODE_INVALID;
1235	pVM->pgm.s.GCPhys4MBPSEMask = RT_BIT_64(32) - 1; /* default; checked later */
1236	pVM->pgm.s.GCPtrPrevRamRangeMapping = MM_HYPER_AREA_ADDRESS;
1237
1238	rc = CFGMR3QueryBoolDef(CFGMR3GetRoot(pVM), "RamPreAlloc", &pVM->pgm.s.fRamPreAlloc,
1239	#ifdef VBOX_WITH_PREALLOC_RAM_BY_DEFAULT
1240	true
1241	#else
1242	false
1243	#endif
1244	);
1245	AssertLogRelRCReturn(rc, rc);
1246
1247	#if HC_ARCH_BITS == 64 \|\| 1 /** @todo 4GB/32-bit: remove \|\| 1 later and adjust the limit. */
1248	rc = CFGMR3QueryU32Def(pCfgPGM, "MaxRing3Chunks", &pVM->pgm.s.ChunkR3Map.cMax, UINT32_MAX);
1249	#else
1250	rc = CFGMR3QueryU32Def(pCfgPGM, "MaxRing3Chunks", &pVM->pgm.s.ChunkR3Map.cMax, _1G / GMM_CHUNK_SIZE);
1251	#endif
1252	AssertLogRelRCReturn(rc, rc);
1253	for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
1254	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
1255
1256	/*
1257	* Get the configured RAM size - to estimate saved state size.
1258	*/
1259	uint64_t cbRam;
1260	rc = CFGMR3QueryU64(CFGMR3GetRoot(pVM), "RamSize", &cbRam);
1261	if (rc == VERR_CFGM_VALUE_NOT_FOUND)
1262	cbRam = 0;
1263	else if (RT_SUCCESS(rc))
1264	{
1265	if (cbRam < PAGE_SIZE)
1266	cbRam = 0;
1267	cbRam = RT_ALIGN_64(cbRam, PAGE_SIZE);
1268	}
1269	else
1270	{
1271	AssertMsgFailed(("Configuration error: Failed to query integer \"RamSize\", rc=%Rrc.\n", rc));
1272	return rc;
1273	}
1274
1275	/*
1276	* Register callbacks, string formatters and the saved state data unit.
1277	*/
1278	#ifdef VBOX_STRICT
1279	VMR3AtStateRegister(pVM, pgmR3ResetNoMorePhysWritesFlag, NULL);
1280	#endif
1281	PGMRegisterStringFormatTypes();
1282
1283	rc = pgmR3InitSavedState(pVM, cbRam);
1284	if (RT_FAILURE(rc))
1285	return rc;
1286
1287	/*
1288	* Initialize the PGM critical section and flush the phys TLBs
1289	*/
1290	rc = PDMR3CritSectInit(pVM, &pVM->pgm.s.CritSect, RT_SRC_POS, "PGM");
1291	AssertRCReturn(rc, rc);
1292
1293	PGMR3PhysChunkInvalidateTLB(pVM);
1294	PGMPhysInvalidatePageMapTLB(pVM);
1295
1296	/*
1297	* For the time being we sport a full set of handy pages in addition to the base
1298	* memory to simplify things.
1299	*/
1300	rc = MMR3ReserveHandyPages(pVM, RT_ELEMENTS(pVM->pgm.s.aHandyPages)); /** @todo this should be changed to PGM_HANDY_PAGES_MIN but this needs proper testing... */
1301	AssertRCReturn(rc, rc);
1302
1303	/*
1304	* Trees
1305	*/
1306	rc = MMHyperAlloc(pVM, sizeof(PGMTREES), 0, MM_TAG_PGM, (void **)&pVM->pgm.s.pTreesR3);
1307	if (RT_SUCCESS(rc))
1308	{
1309	pVM->pgm.s.pTreesR0 = MMHyperR3ToR0(pVM, pVM->pgm.s.pTreesR3);
1310	pVM->pgm.s.pTreesRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pTreesR3);
1311
1312	/*
1313	* Alocate the zero page.
1314	*/
1315	rc = MMHyperAlloc(pVM, PAGE_SIZE, PAGE_SIZE, MM_TAG_PGM, &pVM->pgm.s.pvZeroPgR3);
1316	}
1317	if (RT_SUCCESS(rc))
1318	{
1319	pVM->pgm.s.pvZeroPgRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pvZeroPgR3);
1320	pVM->pgm.s.pvZeroPgR0 = MMHyperR3ToR0(pVM, pVM->pgm.s.pvZeroPgR3);
1321	pVM->pgm.s.HCPhysZeroPg = MMR3HyperHCVirt2HCPhys(pVM, pVM->pgm.s.pvZeroPgR3);
1322	AssertRelease(pVM->pgm.s.HCPhysZeroPg != NIL_RTHCPHYS);
1323
1324	/*
1325	* Init the paging.
1326	*/
1327	rc = pgmR3InitPaging(pVM);
1328	}
1329	if (RT_SUCCESS(rc))
1330	{
1331	/*
1332	* Init the page pool.
1333	*/
1334	rc = pgmR3PoolInit(pVM);
1335	}
1336	if (RT_SUCCESS(rc))
1337	{
1338	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1339	{
1340	PVMCPU pVCpu = &pVM->aCpus[i];
1341	rc = PGMR3ChangeMode(pVM, pVCpu, PGMMODE_REAL);
1342	if (RT_FAILURE(rc))
1343	break;
1344	}
1345	}
1346
1347	if (RT_SUCCESS(rc))
1348	{
1349	/*
1350	* Info & statistics
1351	*/
1352	DBGFR3InfoRegisterInternal(pVM, "mode",
1353	"Shows the current paging mode. "
1354	"Recognizes 'all', 'guest', 'shadow' and 'host' as arguments, defaulting to 'all' if nothing's given.",
1355	pgmR3InfoMode);
1356	DBGFR3InfoRegisterInternal(pVM, "pgmcr3",
1357	"Dumps all the entries in the top level paging table. No arguments.",
1358	pgmR3InfoCr3);
1359	DBGFR3InfoRegisterInternal(pVM, "phys",
1360	"Dumps all the physical address ranges. No arguments.",
1361	pgmR3PhysInfo);
1362	DBGFR3InfoRegisterInternal(pVM, "handlers",
1363	"Dumps physical, virtual and hyper virtual handlers. "
1364	"Pass 'phys', 'virt', 'hyper' as argument if only one kind is wanted."
1365	"Add 'nost' if the statistics are unwanted, use together with 'all' or explicit selection.",
1366	pgmR3InfoHandlers);
1367	DBGFR3InfoRegisterInternal(pVM, "mappings",
1368	"Dumps guest mappings.",
1369	pgmR3MapInfo);
1370
1371	pgmR3InitStats(pVM);
1372
1373	#ifdef VBOX_WITH_DEBUGGER
1374	/*
1375	* Debugger commands.
1376	*/
1377	static bool s_fRegisteredCmds = false;
1378	if (!s_fRegisteredCmds)
1379	{
1380	int rc2 = DBGCRegisterCommands(&g_aCmds[0], RT_ELEMENTS(g_aCmds));
1381	if (RT_SUCCESS(rc2))
1382	s_fRegisteredCmds = true;
1383	}
1384	#endif
1385	return VINF_SUCCESS;
1386	}
1387
1388	/* Almost no cleanup necessary, MM frees all memory. */
1389	PDMR3CritSectDelete(&pVM->pgm.s.CritSect);
1390
1391	return rc;
1392	}
1393
1394
1395	/**
1396	* Initializes the per-VCPU PGM.
1397	*
1398	* @returns VBox status code.
1399	* @param pVM The VM to operate on.
1400	*/
1401	VMMR3DECL(int) PGMR3InitCPU(PVM pVM)
1402	{
1403	LogFlow(("PGMR3InitCPU\n"));
1404	return VINF_SUCCESS;
1405	}
1406
1407
1408	/**
1409	* Init paging.
1410	*
1411	* Since we need to check what mode the host is operating in before we can choose
1412	* the right paging functions for the host we have to delay this until R0 has
1413	* been initialized.
1414	*
1415	* @returns VBox status code.
1416	* @param pVM VM handle.
1417	*/
1418	static int pgmR3InitPaging(PVM pVM)
1419	{
1420	/*
1421	* Force a recalculation of modes and switcher so everyone gets notified.
1422	*/
1423	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1424	{
1425	PVMCPU pVCpu = &pVM->aCpus[i];
1426
1427	pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID;
1428	pVCpu->pgm.s.enmGuestMode = PGMMODE_INVALID;
1429	}
1430
1431	pVM->pgm.s.enmHostMode = SUPPAGINGMODE_INVALID;
1432
1433	/*
1434	* Allocate static mapping space for whatever the cr3 register
1435	* points to and in the case of PAE mode to the 4 PDs.
1436	*/
1437	int rc = MMR3HyperReserve(pVM, PAGE_SIZE * 5, "CR3 mapping", &pVM->pgm.s.GCPtrCR3Mapping);
1438	if (RT_FAILURE(rc))
1439	{
1440	AssertMsgFailed(("Failed to reserve two pages for cr mapping in HMA, rc=%Rrc\n", rc));
1441	return rc;
1442	}
1443	MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
1444
1445	/*
1446	* Allocate pages for the three possible intermediate contexts
1447	* (AMD64, PAE and plain 32-Bit). We maintain all three contexts
1448	* for the sake of simplicity. The AMD64 uses the PAE for the
1449	* lower levels, making the total number of pages 11 (3 + 7 + 1).
1450	*
1451	* We assume that two page tables will be enought for the core code
1452	* mappings (HC virtual and identity).
1453	*/
1454	pVM->pgm.s.pInterPD = (PX86PD)MMR3PageAllocLow(pVM); AssertReturn(pVM->pgm.s.pInterPD, VERR_NO_PAGE_MEMORY);
1455	pVM->pgm.s.apInterPTs[0] = (PX86PT)MMR3PageAllocLow(pVM); AssertReturn(pVM->pgm.s.apInterPTs[0], VERR_NO_PAGE_MEMORY);
1456	pVM->pgm.s.apInterPTs[1] = (PX86PT)MMR3PageAllocLow(pVM); AssertReturn(pVM->pgm.s.apInterPTs[1], VERR_NO_PAGE_MEMORY);
1457	pVM->pgm.s.apInterPaePTs[0] = (PX86PTPAE)MMR3PageAlloc(pVM); AssertReturn(pVM->pgm.s.apInterPaePTs[0], VERR_NO_PAGE_MEMORY);
1458	pVM->pgm.s.apInterPaePTs[1] = (PX86PTPAE)MMR3PageAlloc(pVM); AssertReturn(pVM->pgm.s.apInterPaePTs[1], VERR_NO_PAGE_MEMORY);
1459	pVM->pgm.s.apInterPaePDs[0] = (PX86PDPAE)MMR3PageAlloc(pVM); AssertReturn(pVM->pgm.s.apInterPaePDs[0], VERR_NO_PAGE_MEMORY);
1460	pVM->pgm.s.apInterPaePDs[1] = (PX86PDPAE)MMR3PageAlloc(pVM); AssertReturn(pVM->pgm.s.apInterPaePDs[1], VERR_NO_PAGE_MEMORY);
1461	pVM->pgm.s.apInterPaePDs[2] = (PX86PDPAE)MMR3PageAlloc(pVM); AssertReturn(pVM->pgm.s.apInterPaePDs[2], VERR_NO_PAGE_MEMORY);
1462	pVM->pgm.s.apInterPaePDs[3] = (PX86PDPAE)MMR3PageAlloc(pVM); AssertReturn(pVM->pgm.s.apInterPaePDs[3], VERR_NO_PAGE_MEMORY);
1463	pVM->pgm.s.pInterPaePDPT = (PX86PDPT)MMR3PageAllocLow(pVM); AssertReturn(pVM->pgm.s.pInterPaePDPT, VERR_NO_PAGE_MEMORY);
1464	pVM->pgm.s.pInterPaePDPT64 = (PX86PDPT)MMR3PageAllocLow(pVM); AssertReturn(pVM->pgm.s.pInterPaePDPT64, VERR_NO_PAGE_MEMORY);
1465	pVM->pgm.s.pInterPaePML4 = (PX86PML4)MMR3PageAllocLow(pVM); AssertReturn(pVM->pgm.s.pInterPaePML4, VERR_NO_PAGE_MEMORY);
1466
1467	pVM->pgm.s.HCPhysInterPD = MMPage2Phys(pVM, pVM->pgm.s.pInterPD);
1468	AssertRelease(pVM->pgm.s.HCPhysInterPD != NIL_RTHCPHYS && !(pVM->pgm.s.HCPhysInterPD & PAGE_OFFSET_MASK));
1469	pVM->pgm.s.HCPhysInterPaePDPT = MMPage2Phys(pVM, pVM->pgm.s.pInterPaePDPT);
1470	AssertRelease(pVM->pgm.s.HCPhysInterPaePDPT != NIL_RTHCPHYS && !(pVM->pgm.s.HCPhysInterPaePDPT & PAGE_OFFSET_MASK));
1471	pVM->pgm.s.HCPhysInterPaePML4 = MMPage2Phys(pVM, pVM->pgm.s.pInterPaePML4);
1472	AssertRelease(pVM->pgm.s.HCPhysInterPaePML4 != NIL_RTHCPHYS && !(pVM->pgm.s.HCPhysInterPaePML4 & PAGE_OFFSET_MASK) && pVM->pgm.s.HCPhysInterPaePML4 < 0xffffffff);
1473
1474	/*
1475	* Initialize the pages, setting up the PML4 and PDPT for repetitive 4GB action.
1476	*/
1477	ASMMemZeroPage(pVM->pgm.s.pInterPD);
1478	ASMMemZeroPage(pVM->pgm.s.apInterPTs[0]);
1479	ASMMemZeroPage(pVM->pgm.s.apInterPTs[1]);
1480
1481	ASMMemZeroPage(pVM->pgm.s.apInterPaePTs[0]);
1482	ASMMemZeroPage(pVM->pgm.s.apInterPaePTs[1]);
1483
1484	ASMMemZeroPage(pVM->pgm.s.pInterPaePDPT);
1485	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.apInterPaePDs); i++)
1486	{
1487	ASMMemZeroPage(pVM->pgm.s.apInterPaePDs[i]);
1488	pVM->pgm.s.pInterPaePDPT->a[i].u = X86_PDPE_P \| PGM_PLXFLAGS_PERMANENT
1489	\| MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[i]);
1490	}
1491
1492	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.pInterPaePDPT64->a); i++)
1493	{
1494	const unsigned iPD = i % RT_ELEMENTS(pVM->pgm.s.apInterPaePDs);
1495	pVM->pgm.s.pInterPaePDPT64->a[i].u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A \| PGM_PLXFLAGS_PERMANENT
1496	\| MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[iPD]);
1497	}
1498
1499	RTHCPHYS HCPhysInterPaePDPT64 = MMPage2Phys(pVM, pVM->pgm.s.pInterPaePDPT64);
1500	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.pInterPaePML4->a); i++)
1501	pVM->pgm.s.pInterPaePML4->a[i].u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A \| PGM_PLXFLAGS_PERMANENT
1502	\| HCPhysInterPaePDPT64;
1503
1504	/*
1505	* Initialize paging workers and mode from current host mode
1506	* and the guest running in real mode.
1507	*/
1508	pVM->pgm.s.enmHostMode = SUPR3GetPagingMode();
1509	switch (pVM->pgm.s.enmHostMode)
1510	{
1511	case SUPPAGINGMODE_32_BIT:
1512	case SUPPAGINGMODE_32_BIT_GLOBAL:
1513	case SUPPAGINGMODE_PAE:
1514	case SUPPAGINGMODE_PAE_GLOBAL:
1515	case SUPPAGINGMODE_PAE_NX:
1516	case SUPPAGINGMODE_PAE_GLOBAL_NX:
1517	break;
1518
1519	case SUPPAGINGMODE_AMD64:
1520	case SUPPAGINGMODE_AMD64_GLOBAL:
1521	case SUPPAGINGMODE_AMD64_NX:
1522	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
1523	#ifndef VBOX_WITH_HYBRID_32BIT_KERNEL
1524	if (ARCH_BITS != 64)
1525	{
1526	AssertMsgFailed(("Host mode %d (64-bit) is not supported by non-64bit builds\n", pVM->pgm.s.enmHostMode));
1527	LogRel(("Host mode %d (64-bit) is not supported by non-64bit builds\n", pVM->pgm.s.enmHostMode));
1528	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
1529	}
1530	#endif
1531	break;
1532	default:
1533	AssertMsgFailed(("Host mode %d is not supported\n", pVM->pgm.s.enmHostMode));
1534	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
1535	}
1536	rc = pgmR3ModeDataInit(pVM, false /* don't resolve GC and R0 syms yet */);
1537	if (RT_SUCCESS(rc))
1538	{
1539	LogFlow(("pgmR3InitPaging: returns successfully\n"));
1540	#if HC_ARCH_BITS == 64
1541	LogRel(("Debug: HCPhysInterPD=%RHp HCPhysInterPaePDPT=%RHp HCPhysInterPaePML4=%RHp\n",
1542	pVM->pgm.s.HCPhysInterPD, pVM->pgm.s.HCPhysInterPaePDPT, pVM->pgm.s.HCPhysInterPaePML4));
1543	LogRel(("Debug: apInterPTs={%RHp,%RHp} apInterPaePTs={%RHp,%RHp} apInterPaePDs={%RHp,%RHp,%RHp,%RHp} pInterPaePDPT64=%RHp\n",
1544	MMPage2Phys(pVM, pVM->pgm.s.apInterPTs[0]), MMPage2Phys(pVM, pVM->pgm.s.apInterPTs[1]),
1545	MMPage2Phys(pVM, pVM->pgm.s.apInterPaePTs[0]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePTs[1]),
1546	MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[0]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[1]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[2]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[3]),
1547	MMPage2Phys(pVM, pVM->pgm.s.pInterPaePDPT64)));
1548	#endif
1549
1550	return VINF_SUCCESS;
1551	}
1552
1553	LogFlow(("pgmR3InitPaging: returns %Rrc\n", rc));
1554	return rc;
1555	}
1556
1557
1558	/**
1559	* Init statistics
1560	*/
1561	static void pgmR3InitStats(PVM pVM)
1562	{
1563	PPGM pPGM = &pVM->pgm.s;
1564	int rc;
1565
1566	/* Common - misc variables */
1567	STAM_REL_REG(pVM, &pPGM->cAllPages, STAMTYPE_U32, "/PGM/Page/cAllPages", STAMUNIT_COUNT, "The total number of pages.");
1568	STAM_REL_REG(pVM, &pPGM->cPrivatePages, STAMTYPE_U32, "/PGM/Page/cPrivatePages", STAMUNIT_COUNT, "The number of private pages.");
1569	STAM_REL_REG(pVM, &pPGM->cSharedPages, STAMTYPE_U32, "/PGM/Page/cSharedPages", STAMUNIT_COUNT, "The number of shared pages.");
1570	STAM_REL_REG(pVM, &pPGM->cZeroPages, STAMTYPE_U32, "/PGM/Page/cZeroPages", STAMUNIT_COUNT, "The number of zero backed pages.");
1571	STAM_REL_REG(pVM, &pPGM->cPureMmioPages, STAMTYPE_U32, "/PGM/Page/cPureMmioPages", STAMUNIT_COUNT, "The number of pure MMIO pages.");
1572	STAM_REL_REG(pVM, &pPGM->cMonitoredPages, STAMTYPE_U32, "/PGM/Page/cMonitoredPages", STAMUNIT_COUNT, "The number of write monitored pages.");
1573	STAM_REL_REG(pVM, &pPGM->cWrittenToPages, STAMTYPE_U32, "/PGM/Page/cWrittenToPages", STAMUNIT_COUNT, "The number of previously write monitored pages that have been written to.");
1574	STAM_REL_REG(pVM, &pPGM->cWriteLockedPages, STAMTYPE_U32, "/PGM/Page/cWriteLockedPages", STAMUNIT_COUNT, "The number of write(/read) locked pages.");
1575	STAM_REL_REG(pVM, &pPGM->cReadLockedPages, STAMTYPE_U32, "/PGM/Page/cReadLockedPages", STAMUNIT_COUNT, "The number of read (only) locked pages.");
1576	STAM_REL_REG(pVM, &pPGM->cBalloonedPages, STAMTYPE_U32, "/PGM/Page/cBalloonedPages", STAMUNIT_COUNT, "The number of ballooned pages.");
1577	STAM_REL_REG(pVM, &pPGM->cHandyPages, STAMTYPE_U32, "/PGM/Page/cHandyPages", STAMUNIT_COUNT, "The number of handy pages (not included in cAllPages).");
1578	STAM_REL_REG(pVM, &pPGM->cRelocations, STAMTYPE_COUNTER, "/PGM/cRelocations", STAMUNIT_OCCURENCES,"Number of hypervisor relocations.");
1579	STAM_REL_REG(pVM, &pPGM->ChunkR3Map.c, STAMTYPE_U32, "/PGM/ChunkR3Map/c", STAMUNIT_COUNT, "Number of mapped chunks.");
1580	STAM_REL_REG(pVM, &pPGM->ChunkR3Map.cMax, STAMTYPE_U32, "/PGM/ChunkR3Map/cMax", STAMUNIT_COUNT, "Maximum number of mapped chunks.");
1581
1582	STAM_REL_REG(pVM, &pPGM->StatLargePageAlloc, STAMTYPE_COUNTER, "/PGM/LargePage/Alloc", STAMUNIT_OCCURENCES, "The number of large pages we've used.");
1583	STAM_REL_REG(pVM, &pPGM->StatLargePageReused, STAMTYPE_COUNTER, "/PGM/LargePage/Reused", STAMUNIT_OCCURENCES, "The number of times we've reused a large page.");
1584	STAM_REL_REG(pVM, &pPGM->StatLargePageRefused, STAMTYPE_COUNTER, "/PGM/LargePage/Refused", STAMUNIT_OCCURENCES, "The number of times we couldn't use a large page.");
1585	STAM_REL_REG(pVM, &pPGM->StatLargePageRecheck, STAMTYPE_COUNTER, "/PGM/LargePage/Recheck", STAMUNIT_OCCURENCES, "The number of times we've rechecked a disabled large page.");
1586
1587	/* Live save */
1588	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.fActive, STAMTYPE_U8, "/PGM/LiveSave/fActive", STAMUNIT_COUNT, "Active or not.");
1589	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cIgnoredPages, STAMTYPE_U32, "/PGM/LiveSave/cIgnoredPages", STAMUNIT_COUNT, "The number of ignored pages in the RAM ranges (i.e. MMIO, MMIO2 and ROM).");
1590	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cDirtyPagesLong, STAMTYPE_U32, "/PGM/LiveSave/cDirtyPagesLong", STAMUNIT_COUNT, "Longer term dirty page average.");
1591	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cDirtyPagesShort, STAMTYPE_U32, "/PGM/LiveSave/cDirtyPagesShort", STAMUNIT_COUNT, "Short term dirty page average.");
1592	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cPagesPerSecond, STAMTYPE_U32, "/PGM/LiveSave/cPagesPerSecond", STAMUNIT_COUNT, "Pages per second.");
1593	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cSavedPages, STAMTYPE_U64, "/PGM/LiveSave/cSavedPages", STAMUNIT_COUNT, "The total number of saved pages.");
1594	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Ram.cReadyPages, STAMTYPE_U32, "/PGM/LiveSave/Ram/cReadPages", STAMUNIT_COUNT, "RAM: Ready pages.");
1595	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Ram.cDirtyPages, STAMTYPE_U32, "/PGM/LiveSave/Ram/cDirtyPages", STAMUNIT_COUNT, "RAM: Dirty pages.");
1596	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Ram.cZeroPages, STAMTYPE_U32, "/PGM/LiveSave/Ram/cZeroPages", STAMUNIT_COUNT, "RAM: Ready zero pages.");
1597	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Ram.cMonitoredPages, STAMTYPE_U32, "/PGM/LiveSave/Ram/cMonitoredPages", STAMUNIT_COUNT, "RAM: Write monitored pages.");
1598	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Rom.cReadyPages, STAMTYPE_U32, "/PGM/LiveSave/Rom/cReadPages", STAMUNIT_COUNT, "ROM: Ready pages.");
1599	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Rom.cDirtyPages, STAMTYPE_U32, "/PGM/LiveSave/Rom/cDirtyPages", STAMUNIT_COUNT, "ROM: Dirty pages.");
1600	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Rom.cZeroPages, STAMTYPE_U32, "/PGM/LiveSave/Rom/cZeroPages", STAMUNIT_COUNT, "ROM: Ready zero pages.");
1601	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Rom.cMonitoredPages, STAMTYPE_U32, "/PGM/LiveSave/Rom/cMonitoredPages", STAMUNIT_COUNT, "ROM: Write monitored pages.");
1602	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Mmio2.cReadyPages, STAMTYPE_U32, "/PGM/LiveSave/Mmio2/cReadPages", STAMUNIT_COUNT, "MMIO2: Ready pages.");
1603	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Mmio2.cDirtyPages, STAMTYPE_U32, "/PGM/LiveSave/Mmio2/cDirtyPages", STAMUNIT_COUNT, "MMIO2: Dirty pages.");
1604	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Mmio2.cZeroPages, STAMTYPE_U32, "/PGM/LiveSave/Mmio2/cZeroPages", STAMUNIT_COUNT, "MMIO2: Ready zero pages.");
1605	STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Mmio2.cMonitoredPages,STAMTYPE_U32, "/PGM/LiveSave/Mmio2/cMonitoredPages",STAMUNIT_COUNT, "MMIO2: Write monitored pages.");
1606
1607	#ifdef VBOX_WITH_STATISTICS
1608
1609	# define PGM_REG_COUNTER(a, b, c) \
1610	rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, c, b); \
1611	AssertRC(rc);
1612
1613	# define PGM_REG_COUNTER_BYTES(a, b, c) \
1614	rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, c, b); \
1615	AssertRC(rc);
1616
1617	# define PGM_REG_PROFILE(a, b, c) \
1618	rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b); \
1619	AssertRC(rc);
1620
1621	PGM_REG_PROFILE(&pPGM->StatAllocLargePage, "/PGM/LargePage/Prof/Alloc", "Time spent by the host OS for large page allocation.");
1622	PGM_REG_PROFILE(&pPGM->StatClearLargePage, "/PGM/LargePage/Prof/Clear", "Time spent clearing the newly allocated large pages.");
1623	PGM_REG_PROFILE(&pPGM->StatR3IsValidLargePage, "/PGM/LargePage/Prof/R3/IsValid", "pgmPhysIsValidLargePage profiling - R3.");
1624	PGM_REG_PROFILE(&pPGM->StatRZIsValidLargePage, "/PGM/LargePage/Prof/RZ/IsValid", "pgmPhysIsValidLargePage profiling - RZ.");
1625
1626	PGM_REG_COUNTER(&pPGM->StatR3DetectedConflicts, "/PGM/R3/DetectedConflicts", "The number of times PGMR3CheckMappingConflicts() detected a conflict.");
1627	PGM_REG_PROFILE(&pPGM->StatR3ResolveConflict, "/PGM/R3/ResolveConflict", "pgmR3SyncPTResolveConflict() profiling (includes the entire relocation).");
1628	PGM_REG_COUNTER(&pPGM->StatR3PhysRead, "/PGM/R3/Phys/Read", "The number of times PGMPhysRead was called.");
1629	PGM_REG_COUNTER_BYTES(&pPGM->StatR3PhysReadBytes, "/PGM/R3/Phys/Read/Bytes", "The number of bytes read by PGMPhysRead.");
1630	PGM_REG_COUNTER(&pPGM->StatR3PhysWrite, "/PGM/R3/Phys/Write", "The number of times PGMPhysWrite was called.");
1631	PGM_REG_COUNTER_BYTES(&pPGM->StatR3PhysWriteBytes, "/PGM/R3/Phys/Write/Bytes", "The number of bytes written by PGMPhysWrite.");
1632	PGM_REG_COUNTER(&pPGM->StatR3PhysSimpleRead, "/PGM/R3/Phys/Simple/Read", "The number of times PGMPhysSimpleReadGCPtr was called.");
1633	PGM_REG_COUNTER_BYTES(&pPGM->StatR3PhysSimpleReadBytes, "/PGM/R3/Phys/Simple/Read/Bytes", "The number of bytes read by PGMPhysSimpleReadGCPtr.");
1634	PGM_REG_COUNTER(&pPGM->StatR3PhysSimpleWrite, "/PGM/R3/Phys/Simple/Write", "The number of times PGMPhysSimpleWriteGCPtr was called.");
1635	PGM_REG_COUNTER_BYTES(&pPGM->StatR3PhysSimpleWriteBytes, "/PGM/R3/Phys/Simple/Write/Bytes", "The number of bytes written by PGMPhysSimpleWriteGCPtr.");
1636
1637	PGM_REG_COUNTER(&pPGM->StatRZChunkR3MapTlbHits, "/PGM/ChunkR3Map/TlbHitsRZ", "TLB hits.");
1638	PGM_REG_COUNTER(&pPGM->StatRZChunkR3MapTlbMisses, "/PGM/ChunkR3Map/TlbMissesRZ", "TLB misses.");
1639	PGM_REG_COUNTER(&pPGM->StatRZPageMapTlbHits, "/PGM/RZ/Page/MapTlbHits", "TLB hits.");
1640	PGM_REG_COUNTER(&pPGM->StatRZPageMapTlbMisses, "/PGM/RZ/Page/MapTlbMisses", "TLB misses.");
1641	PGM_REG_COUNTER(&pPGM->StatR3ChunkR3MapTlbHits, "/PGM/ChunkR3Map/TlbHitsR3", "TLB hits.");
1642	PGM_REG_COUNTER(&pPGM->StatR3ChunkR3MapTlbMisses, "/PGM/ChunkR3Map/TlbMissesR3", "TLB misses.");
1643	PGM_REG_COUNTER(&pPGM->StatR3PageMapTlbHits, "/PGM/R3/Page/MapTlbHits", "TLB hits.");
1644	PGM_REG_COUNTER(&pPGM->StatR3PageMapTlbMisses, "/PGM/R3/Page/MapTlbMisses", "TLB misses.");
1645	PGM_REG_COUNTER(&pPGM->StatPageMapTlbFlushes, "/PGM/R3/Page/MapTlbFlushes", "TLB flushes (all contexts).");
1646	PGM_REG_COUNTER(&pPGM->StatPageMapTlbFlushEntry, "/PGM/R3/Page/MapTlbFlushEntry", "TLB entry flushes (all contexts).");
1647
1648	PGM_REG_PROFILE(&pPGM->StatRZSyncCR3HandlerVirtualUpdate, "/PGM/RZ/SyncCR3/Handlers/VirtualUpdate", "Profiling of the virtual handler updates.");
1649	PGM_REG_PROFILE(&pPGM->StatRZSyncCR3HandlerVirtualReset, "/PGM/RZ/SyncCR3/Handlers/VirtualReset", "Profiling of the virtual handler resets.");
1650	PGM_REG_PROFILE(&pPGM->StatR3SyncCR3HandlerVirtualUpdate, "/PGM/R3/SyncCR3/Handlers/VirtualUpdate", "Profiling of the virtual handler updates.");
1651	PGM_REG_PROFILE(&pPGM->StatR3SyncCR3HandlerVirtualReset, "/PGM/R3/SyncCR3/Handlers/VirtualReset", "Profiling of the virtual handler resets.");
1652
1653	PGM_REG_COUNTER(&pPGM->StatRZPhysHandlerReset, "/PGM/RZ/PhysHandlerReset", "The number of times PGMHandlerPhysicalReset is called.");
1654	PGM_REG_COUNTER(&pPGM->StatR3PhysHandlerReset, "/PGM/R3/PhysHandlerReset", "The number of times PGMHandlerPhysicalReset is called.");
1655	PGM_REG_PROFILE(&pPGM->StatRZVirtHandlerSearchByPhys, "/PGM/RZ/VirtHandlerSearchByPhys", "Profiling of pgmHandlerVirtualFindByPhysAddr.");
1656	PGM_REG_PROFILE(&pPGM->StatR3VirtHandlerSearchByPhys, "/PGM/R3/VirtHandlerSearchByPhys", "Profiling of pgmHandlerVirtualFindByPhysAddr.");
1657
1658	PGM_REG_COUNTER(&pPGM->StatRZPageReplaceShared, "/PGM/RZ/Page/ReplacedShared", "Times a shared page was replaced.");
1659	PGM_REG_COUNTER(&pPGM->StatRZPageReplaceZero, "/PGM/RZ/Page/ReplacedZero", "Times the zero page was replaced.");
1660	/// @todo PGM_REG_COUNTER(&pPGM->StatRZPageHandyAllocs, "/PGM/RZ/Page/HandyAllocs", "Number of times we've allocated more handy pages.");
1661	PGM_REG_COUNTER(&pPGM->StatR3PageReplaceShared, "/PGM/R3/Page/ReplacedShared", "Times a shared page was replaced.");
1662	PGM_REG_COUNTER(&pPGM->StatR3PageReplaceZero, "/PGM/R3/Page/ReplacedZero", "Times the zero page was replaced.");
1663	/// @todo PGM_REG_COUNTER(&pPGM->StatR3PageHandyAllocs, "/PGM/R3/Page/HandyAllocs", "Number of times we've allocated more handy pages.");
1664
1665	PGM_REG_COUNTER(&pPGM->StatRZPhysRead, "/PGM/RZ/Phys/Read", "The number of times PGMPhysRead was called.");
1666	PGM_REG_COUNTER_BYTES(&pPGM->StatRZPhysReadBytes, "/PGM/RZ/Phys/Read/Bytes", "The number of bytes read by PGMPhysRead.");
1667	PGM_REG_COUNTER(&pPGM->StatRZPhysWrite, "/PGM/RZ/Phys/Write", "The number of times PGMPhysWrite was called.");
1668	PGM_REG_COUNTER_BYTES(&pPGM->StatRZPhysWriteBytes, "/PGM/RZ/Phys/Write/Bytes", "The number of bytes written by PGMPhysWrite.");
1669	PGM_REG_COUNTER(&pPGM->StatRZPhysSimpleRead, "/PGM/RZ/Phys/Simple/Read", "The number of times PGMPhysSimpleReadGCPtr was called.");
1670	PGM_REG_COUNTER_BYTES(&pPGM->StatRZPhysSimpleReadBytes, "/PGM/RZ/Phys/Simple/Read/Bytes", "The number of bytes read by PGMPhysSimpleReadGCPtr.");
1671	PGM_REG_COUNTER(&pPGM->StatRZPhysSimpleWrite, "/PGM/RZ/Phys/Simple/Write", "The number of times PGMPhysSimpleWriteGCPtr was called.");
1672	PGM_REG_COUNTER_BYTES(&pPGM->StatRZPhysSimpleWriteBytes, "/PGM/RZ/Phys/Simple/Write/Bytes", "The number of bytes written by PGMPhysSimpleWriteGCPtr.");
1673
1674	/* GC only: */
1675	PGM_REG_COUNTER(&pPGM->StatRCDynMapCacheHits, "/PGM/RC/DynMapCache/Hits" , "Number of dynamic page mapping cache hits.");
1676	PGM_REG_COUNTER(&pPGM->StatRCDynMapCacheMisses, "/PGM/RC/DynMapCache/Misses" , "Number of dynamic page mapping cache misses.");
1677	PGM_REG_COUNTER(&pPGM->StatRCInvlPgConflict, "/PGM/RC/InvlPgConflict", "Number of times PGMInvalidatePage() detected a mapping conflict.");
1678	PGM_REG_COUNTER(&pPGM->StatRCInvlPgSyncMonCR3, "/PGM/RC/InvlPgSyncMonitorCR3", "Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3.");
1679
1680	PGM_REG_COUNTER(&pPGM->StatRCPhysRead, "/PGM/RC/Phys/Read", "The number of times PGMPhysRead was called.");
1681	PGM_REG_COUNTER_BYTES(&pPGM->StatRCPhysReadBytes, "/PGM/RC/Phys/Read/Bytes", "The number of bytes read by PGMPhysRead.");
1682	PGM_REG_COUNTER(&pPGM->StatRCPhysWrite, "/PGM/RC/Phys/Write", "The number of times PGMPhysWrite was called.");
1683	PGM_REG_COUNTER_BYTES(&pPGM->StatRCPhysWriteBytes, "/PGM/RC/Phys/Write/Bytes", "The number of bytes written by PGMPhysWrite.");
1684	PGM_REG_COUNTER(&pPGM->StatRCPhysSimpleRead, "/PGM/RC/Phys/Simple/Read", "The number of times PGMPhysSimpleReadGCPtr was called.");
1685	PGM_REG_COUNTER_BYTES(&pPGM->StatRCPhysSimpleReadBytes, "/PGM/RC/Phys/Simple/Read/Bytes", "The number of bytes read by PGMPhysSimpleReadGCPtr.");
1686	PGM_REG_COUNTER(&pPGM->StatRCPhysSimpleWrite, "/PGM/RC/Phys/Simple/Write", "The number of times PGMPhysSimpleWriteGCPtr was called.");
1687	PGM_REG_COUNTER_BYTES(&pPGM->StatRCPhysSimpleWriteBytes, "/PGM/RC/Phys/Simple/Write/Bytes", "The number of bytes written by PGMPhysSimpleWriteGCPtr.");
1688
1689	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1690	PGM_REG_COUNTER(&pPGM->StatTrackVirgin, "/PGM/Track/Virgin", "The number of first time shadowings");
1691	PGM_REG_COUNTER(&pPGM->StatTrackAliased, "/PGM/Track/Aliased", "The number of times switching to cRef2, i.e. the page is being shadowed by two PTs.");
1692	PGM_REG_COUNTER(&pPGM->StatTrackAliasedMany, "/PGM/Track/AliasedMany", "The number of times we're tracking using cRef2.");
1693	PGM_REG_COUNTER(&pPGM->StatTrackAliasedLots, "/PGM/Track/AliasedLots", "The number of times we're hitting pages which has overflowed cRef2");
1694	PGM_REG_COUNTER(&pPGM->StatTrackOverflows, "/PGM/Track/Overflows", "The number of times the extent list grows too long.");
1695	PGM_REG_PROFILE(&pPGM->StatTrackDeref, "/PGM/Track/Deref", "Profiling of SyncPageWorkerTrackDeref (expensive).");
1696	# endif
1697
1698	# undef PGM_REG_COUNTER
1699	# undef PGM_REG_PROFILE
1700	#endif
1701
1702	/*
1703	* Note! The layout below matches the member layout exactly!
1704	*/
1705
1706	/*
1707	* Common - stats
1708	*/
1709	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1710	{
1711	PPGMCPU pPgmCpu = &pVM->aCpus[idCpu].pgm.s;
1712
1713	#define PGM_REG_COUNTER(a, b, c) \
1714	rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, c, b, idCpu); \
1715	AssertRC(rc);
1716	#define PGM_REG_PROFILE(a, b, c) \
1717	rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b, idCpu); \
1718	AssertRC(rc);
1719
1720	PGM_REG_COUNTER(&pPgmCpu->cGuestModeChanges, "/PGM/CPU%u/cGuestModeChanges", "Number of guest mode changes.");
1721
1722	#ifdef VBOX_WITH_STATISTICS
1723
1724	# if 0 /* rarely useful; leave for debugging. */
1725	for (unsigned j = 0; j < RT_ELEMENTS(pPgmCpu->StatSyncPtPD); j++)
1726	STAMR3RegisterF(pVM, &pPgmCpu->StatSyncPtPD[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
1727	"The number of SyncPT per PD n.", "/PGM/CPU%u/PDSyncPT/%04X", i, j);
1728	for (unsigned j = 0; j < RT_ELEMENTS(pPgmCpu->StatSyncPagePD); j++)
1729	STAMR3RegisterF(pVM, &pPgmCpu->StatSyncPagePD[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
1730	"The number of SyncPage per PD n.", "/PGM/CPU%u/PDSyncPage/%04X", i, j);
1731	# endif
1732	/* R0 only: */
1733	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapMigrateInvlPg, "/PGM/CPU%u/R0/DynMapMigrateInvlPg", "invlpg count in PGMDynMapMigrateAutoSet.");
1734	PGM_REG_PROFILE(&pPgmCpu->StatR0DynMapGCPageInl, "/PGM/CPU%u/R0/DynMapPageGCPageInl", "Calls to pgmR0DynMapGCPageInlined.");
1735	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapGCPageInlHits, "/PGM/CPU%u/R0/DynMapPageGCPageInl/Hits", "Hash table lookup hits.");
1736	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapGCPageInlMisses, "/PGM/CPU%u/R0/DynMapPageGCPageInl/Misses", "Misses that falls back to code common with PGMDynMapHCPage.");
1737	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapGCPageInlRamHits, "/PGM/CPU%u/R0/DynMapPageGCPageInl/RamHits", "1st ram range hits.");
1738	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapGCPageInlRamMisses, "/PGM/CPU%u/R0/DynMapPageGCPageInl/RamMisses", "1st ram range misses, takes slow path.");
1739	PGM_REG_PROFILE(&pPgmCpu->StatR0DynMapHCPageInl, "/PGM/CPU%u/R0/DynMapPageHCPageInl", "Calls to pgmR0DynMapHCPageInlined.");
1740	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapHCPageInlHits, "/PGM/CPU%u/R0/DynMapPageHCPageInl/Hits", "Hash table lookup hits.");
1741	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapHCPageInlMisses, "/PGM/CPU%u/R0/DynMapPageHCPageInl/Misses", "Misses that falls back to code common with PGMDynMapHCPage.");
1742	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapPage, "/PGM/CPU%u/R0/DynMapPage", "Calls to pgmR0DynMapPage");
1743	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapSetOptimize, "/PGM/CPU%u/R0/DynMapPage/SetOptimize", "Calls to pgmDynMapOptimizeAutoSet.");
1744	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapSetSearchFlushes, "/PGM/CPU%u/R0/DynMapPage/SetSearchFlushes","Set search restorting to subset flushes.");
1745	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapSetSearchHits, "/PGM/CPU%u/R0/DynMapPage/SetSearchHits", "Set search hits.");
1746	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapSetSearchMisses, "/PGM/CPU%u/R0/DynMapPage/SetSearchMisses", "Set search misses.");
1747	PGM_REG_PROFILE(&pPgmCpu->StatR0DynMapHCPage, "/PGM/CPU%u/R0/DynMapPage/HCPage", "Calls to PGMDynMapHCPage (ring-0).");
1748	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapPageHits0, "/PGM/CPU%u/R0/DynMapPage/Hits0", "Hits at iPage+0");
1749	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapPageHits1, "/PGM/CPU%u/R0/DynMapPage/Hits1", "Hits at iPage+1");
1750	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapPageHits2, "/PGM/CPU%u/R0/DynMapPage/Hits2", "Hits at iPage+2");
1751	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapPageInvlPg, "/PGM/CPU%u/R0/DynMapPage/InvlPg", "invlpg count in pgmR0DynMapPageSlow.");
1752	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapPageSlow, "/PGM/CPU%u/R0/DynMapPage/Slow", "Calls to pgmR0DynMapPageSlow - subtract this from pgmR0DynMapPage to get 1st level hits.");
1753	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapPageSlowLoopHits, "/PGM/CPU%u/R0/DynMapPage/SlowLoopHits" , "Hits in the loop path.");
1754	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapPageSlowLoopMisses, "/PGM/CPU%u/R0/DynMapPage/SlowLoopMisses", "Misses in the loop path. NonLoopMisses = Slow - SlowLoopHit - SlowLoopMisses");
1755	//PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapPageSlowLostHits, "/PGM/CPU%u/R0/DynMapPage/SlowLostHits", "Lost hits.");
1756	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapSubsets, "/PGM/CPU%u/R0/Subsets", "Times PGMDynMapPushAutoSubset was called.");
1757	PGM_REG_COUNTER(&pPgmCpu->StatR0DynMapPopFlushes, "/PGM/CPU%u/R0/SubsetPopFlushes", "Times PGMDynMapPopAutoSubset flushes the subset.");
1758	PGM_REG_COUNTER(&pPgmCpu->aStatR0DynMapSetSize[0], "/PGM/CPU%u/R0/SetSize000..09", "00-09% filled");
1759	PGM_REG_COUNTER(&pPgmCpu->aStatR0DynMapSetSize[1], "/PGM/CPU%u/R0/SetSize010..19", "10-19% filled");
1760	PGM_REG_COUNTER(&pPgmCpu->aStatR0DynMapSetSize[2], "/PGM/CPU%u/R0/SetSize020..29", "20-29% filled");
1761	PGM_REG_COUNTER(&pPgmCpu->aStatR0DynMapSetSize[3], "/PGM/CPU%u/R0/SetSize030..39", "30-39% filled");
1762	PGM_REG_COUNTER(&pPgmCpu->aStatR0DynMapSetSize[4], "/PGM/CPU%u/R0/SetSize040..49", "40-49% filled");
1763	PGM_REG_COUNTER(&pPgmCpu->aStatR0DynMapSetSize[5], "/PGM/CPU%u/R0/SetSize050..59", "50-59% filled");
1764	PGM_REG_COUNTER(&pPgmCpu->aStatR0DynMapSetSize[6], "/PGM/CPU%u/R0/SetSize060..69", "60-69% filled");
1765	PGM_REG_COUNTER(&pPgmCpu->aStatR0DynMapSetSize[7], "/PGM/CPU%u/R0/SetSize070..79", "70-79% filled");
1766	PGM_REG_COUNTER(&pPgmCpu->aStatR0DynMapSetSize[8], "/PGM/CPU%u/R0/SetSize080..89", "80-89% filled");
1767	PGM_REG_COUNTER(&pPgmCpu->aStatR0DynMapSetSize[9], "/PGM/CPU%u/R0/SetSize090..99", "90-99% filled");
1768	PGM_REG_COUNTER(&pPgmCpu->aStatR0DynMapSetSize[10], "/PGM/CPU%u/R0/SetSize100", "100% filled");
1769
1770	/* RZ only: */
1771	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0e, "/PGM/CPU%u/RZ/Trap0e", "Profiling of the PGMTrap0eHandler() body.");
1772	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTimeCheckPageFault, "/PGM/CPU%u/RZ/Trap0e/Time/CheckPageFault", "Profiling of checking for dirty/access emulation faults.");
1773	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTimeSyncPT, "/PGM/CPU%u/RZ/Trap0e/Time/SyncPT", "Profiling of lazy page table syncing.");
1774	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTimeMapping, "/PGM/CPU%u/RZ/Trap0e/Time/Mapping", "Profiling of checking virtual mappings.");
1775	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTimeOutOfSync, "/PGM/CPU%u/RZ/Trap0e/Time/OutOfSync", "Profiling of out of sync page handling.");
1776	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTimeHandlers, "/PGM/CPU%u/RZ/Trap0e/Time/Handlers", "Profiling of checking handlers.");
1777	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTime2CSAM, "/PGM/CPU%u/RZ/Trap0e/Time2/CSAM", "Profiling of the Trap0eHandler body when the cause is CSAM.");
1778	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTime2DirtyAndAccessed, "/PGM/CPU%u/RZ/Trap0e/Time2/DirtyAndAccessedBits", "Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation.");
1779	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTime2GuestTrap, "/PGM/CPU%u/RZ/Trap0e/Time2/GuestTrap", "Profiling of the Trap0eHandler body when the cause is a guest trap.");
1780	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTime2HndPhys, "/PGM/CPU%u/RZ/Trap0e/Time2/HandlerPhysical", "Profiling of the Trap0eHandler body when the cause is a physical handler.");
1781	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTime2HndVirt, "/PGM/CPU%u/RZ/Trap0e/Time2/HandlerVirtual", "Profiling of the Trap0eHandler body when the cause is a virtual handler.");
1782	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTime2HndUnhandled, "/PGM/CPU%u/RZ/Trap0e/Time2/HandlerUnhandled", "Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page.");
1783	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTime2Misc, "/PGM/CPU%u/RZ/Trap0e/Time2/Misc", "Profiling of the Trap0eHandler body when the cause is not known.");
1784	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTime2OutOfSync, "/PGM/CPU%u/RZ/Trap0e/Time2/OutOfSync", "Profiling of the Trap0eHandler body when the cause is an out-of-sync page.");
1785	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTime2OutOfSyncHndPhys, "/PGM/CPU%u/RZ/Trap0e/Time2/OutOfSyncHndPhys", "Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page.");
1786	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTime2OutOfSyncHndVirt, "/PGM/CPU%u/RZ/Trap0e/Time2/OutOfSyncHndVirt", "Profiling of the Trap0eHandler body when the cause is an out-of-sync virtual handler page.");
1787	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTime2OutOfSyncHndObs, "/PGM/CPU%u/RZ/Trap0e/Time2/OutOfSyncObsHnd", "Profiling of the Trap0eHandler body when the cause is an obsolete handler page.");
1788	PGM_REG_PROFILE(&pPgmCpu->StatRZTrap0eTime2SyncPT, "/PGM/CPU%u/RZ/Trap0e/Time2/SyncPT", "Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT.");
1789	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eConflicts, "/PGM/CPU%u/RZ/Trap0e/Conflicts", "The number of times #PF was caused by an undetected conflict.");
1790	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eHandlersMapping, "/PGM/CPU%u/RZ/Trap0e/Handlers/Mapping", "Number of traps due to access handlers in mappings.");
1791	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eHandlersOutOfSync, "/PGM/CPU%u/RZ/Trap0e/Handlers/OutOfSync", "Number of traps due to out-of-sync handled pages.");
1792	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eHandlersPhysical, "/PGM/CPU%u/RZ/Trap0e/Handlers/Physical", "Number of traps due to physical access handlers.");
1793	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eHandlersVirtual, "/PGM/CPU%u/RZ/Trap0e/Handlers/Virtual", "Number of traps due to virtual access handlers.");
1794	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eHandlersVirtualByPhys, "/PGM/CPU%u/RZ/Trap0e/Handlers/VirtualByPhys", "Number of traps due to virtual access handlers by physical address.");
1795	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eHandlersVirtualUnmarked,"/PGM/CPU%u/RZ/Trap0e/Handlers/VirtualUnmarked","Number of traps due to virtual access handlers by virtual address (without proper physical flags).");
1796	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eHandlersUnhandled, "/PGM/CPU%u/RZ/Trap0e/Handlers/Unhandled", "Number of traps due to access outside range of monitored page(s).");
1797	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eHandlersInvalid, "/PGM/CPU%u/RZ/Trap0e/Handlers/Invalid", "Number of traps due to access to invalid physical memory.");
1798	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eUSNotPresentRead, "/PGM/CPU%u/RZ/Trap0e/Err/User/NPRead", "Number of user mode not present read page faults.");
1799	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eUSNotPresentWrite, "/PGM/CPU%u/RZ/Trap0e/Err/User/NPWrite", "Number of user mode not present write page faults.");
1800	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eUSWrite, "/PGM/CPU%u/RZ/Trap0e/Err/User/Write", "Number of user mode write page faults.");
1801	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eUSReserved, "/PGM/CPU%u/RZ/Trap0e/Err/User/Reserved", "Number of user mode reserved bit page faults.");
1802	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eUSNXE, "/PGM/CPU%u/RZ/Trap0e/Err/User/NXE", "Number of user mode NXE page faults.");
1803	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eUSRead, "/PGM/CPU%u/RZ/Trap0e/Err/User/Read", "Number of user mode read page faults.");
1804	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eSVNotPresentRead, "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/NPRead", "Number of supervisor mode not present read page faults.");
1805	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eSVNotPresentWrite, "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/NPWrite", "Number of supervisor mode not present write page faults.");
1806	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eSVWrite, "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/Write", "Number of supervisor mode write page faults.");
1807	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eSVReserved, "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/Reserved", "Number of supervisor mode reserved bit page faults.");
1808	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eSNXE, "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/NXE", "Number of supervisor mode NXE page faults.");
1809	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eGuestPF, "/PGM/CPU%u/RZ/Trap0e/GuestPF", "Number of real guest page faults.");
1810	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eGuestPFUnh, "/PGM/CPU%u/RZ/Trap0e/GuestPF/Unhandled", "Number of real guest page faults from the 'unhandled' case.");
1811	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eGuestPFMapping, "/PGM/CPU%u/RZ/Trap0e/GuestPF/InMapping", "Number of real guest page faults in a mapping.");
1812	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eWPEmulInRZ, "/PGM/CPU%u/RZ/Trap0e/WP/InRZ", "Number of guest page faults due to X86_CR0_WP emulation.");
1813	PGM_REG_COUNTER(&pPgmCpu->StatRZTrap0eWPEmulToR3, "/PGM/CPU%u/RZ/Trap0e/WP/ToR3", "Number of guest page faults due to X86_CR0_WP emulation (forward to R3 for emulation).");
1814	#if 0 /* rarely useful; leave for debugging. */
1815	for (unsigned j = 0; j < RT_ELEMENTS(pPgmCpu->StatRZTrap0ePD); j++)
1816	STAMR3RegisterF(pVM, &pPgmCpu->StatRZTrap0ePD[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
1817	"The number of traps in page directory n.", "/PGM/CPU%u/RZ/Trap0e/PD/%04X", i, j);
1818	#endif
1819	PGM_REG_COUNTER(&pPgmCpu->StatRZGuestCR3WriteHandled, "/PGM/CPU%u/RZ/CR3WriteHandled", "The number of times the Guest CR3 change was successfully handled.");
1820	PGM_REG_COUNTER(&pPgmCpu->StatRZGuestCR3WriteUnhandled, "/PGM/CPU%u/RZ/CR3WriteUnhandled", "The number of times the Guest CR3 change was passed back to the recompiler.");
1821	PGM_REG_COUNTER(&pPgmCpu->StatRZGuestCR3WriteConflict, "/PGM/CPU%u/RZ/CR3WriteConflict", "The number of times the Guest CR3 monitoring detected a conflict.");
1822	PGM_REG_COUNTER(&pPgmCpu->StatRZGuestROMWriteHandled, "/PGM/CPU%u/RZ/ROMWriteHandled", "The number of times the Guest ROM change was successfully handled.");
1823	PGM_REG_COUNTER(&pPgmCpu->StatRZGuestROMWriteUnhandled, "/PGM/CPU%u/RZ/ROMWriteUnhandled", "The number of times the Guest ROM change was passed back to the recompiler.");
1824
1825	/* HC only: */
1826
1827	/* RZ & R3: */
1828	PGM_REG_PROFILE(&pPgmCpu->StatRZSyncCR3, "/PGM/CPU%u/RZ/SyncCR3", "Profiling of the PGMSyncCR3() body.");
1829	PGM_REG_PROFILE(&pPgmCpu->StatRZSyncCR3Handlers, "/PGM/CPU%u/RZ/SyncCR3/Handlers", "Profiling of the PGMSyncCR3() update handler section.");
1830	PGM_REG_COUNTER(&pPgmCpu->StatRZSyncCR3Global, "/PGM/CPU%u/RZ/SyncCR3/Global", "The number of global CR3 syncs.");
1831	PGM_REG_COUNTER(&pPgmCpu->StatRZSyncCR3NotGlobal, "/PGM/CPU%u/RZ/SyncCR3/NotGlobal", "The number of non-global CR3 syncs.");
1832	PGM_REG_COUNTER(&pPgmCpu->StatRZSyncCR3DstCacheHit, "/PGM/CPU%u/RZ/SyncCR3/DstChacheHit", "The number of times we got some kind of a cache hit.");
1833	PGM_REG_COUNTER(&pPgmCpu->StatRZSyncCR3DstFreed, "/PGM/CPU%u/RZ/SyncCR3/DstFreed", "The number of times we've had to free a shadow entry.");
1834	PGM_REG_COUNTER(&pPgmCpu->StatRZSyncCR3DstFreedSrcNP, "/PGM/CPU%u/RZ/SyncCR3/DstFreedSrcNP", "The number of times we've had to free a shadow entry for which the source entry was not present.");
1835	PGM_REG_COUNTER(&pPgmCpu->StatRZSyncCR3DstNotPresent, "/PGM/CPU%u/RZ/SyncCR3/DstNotPresent", "The number of times we've encountered a not present shadow entry for a present guest entry.");
1836	PGM_REG_COUNTER(&pPgmCpu->StatRZSyncCR3DstSkippedGlobalPD, "/PGM/CPU%u/RZ/SyncCR3/DstSkippedGlobalPD", "The number of times a global page directory wasn't flushed.");
1837	PGM_REG_COUNTER(&pPgmCpu->StatRZSyncCR3DstSkippedGlobalPT, "/PGM/CPU%u/RZ/SyncCR3/DstSkippedGlobalPT", "The number of times a page table with only global entries wasn't flushed.");
1838	PGM_REG_PROFILE(&pPgmCpu->StatRZSyncPT, "/PGM/CPU%u/RZ/SyncPT", "Profiling of the pfnSyncPT() body.");
1839	PGM_REG_COUNTER(&pPgmCpu->StatRZSyncPTFailed, "/PGM/CPU%u/RZ/SyncPT/Failed", "The number of times pfnSyncPT() failed.");
1840	PGM_REG_COUNTER(&pPgmCpu->StatRZSyncPT4K, "/PGM/CPU%u/RZ/SyncPT/4K", "Nr of 4K PT syncs");
1841	PGM_REG_COUNTER(&pPgmCpu->StatRZSyncPT4M, "/PGM/CPU%u/RZ/SyncPT/4M", "Nr of 4M PT syncs");
1842	PGM_REG_COUNTER(&pPgmCpu->StatRZSyncPagePDNAs, "/PGM/CPU%u/RZ/SyncPagePDNAs", "The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit.");
1843	PGM_REG_COUNTER(&pPgmCpu->StatRZSyncPagePDOutOfSync, "/PGM/CPU%u/RZ/SyncPagePDOutOfSync", "The number of time we've encountered an out-of-sync PD in SyncPage.");
1844	PGM_REG_COUNTER(&pPgmCpu->StatRZAccessedPage, "/PGM/CPU%u/RZ/AccessedPage", "The number of pages marked not present for accessed bit emulation.");
1845	PGM_REG_PROFILE(&pPgmCpu->StatRZDirtyBitTracking, "/PGM/CPU%u/RZ/DirtyPage", "Profiling the dirty bit tracking in CheckPageFault().");
1846	PGM_REG_COUNTER(&pPgmCpu->StatRZDirtyPage, "/PGM/CPU%u/RZ/DirtyPage/Mark", "The number of pages marked read-only for dirty bit tracking.");
1847	PGM_REG_COUNTER(&pPgmCpu->StatRZDirtyPageBig, "/PGM/CPU%u/RZ/DirtyPage/MarkBig", "The number of 4MB pages marked read-only for dirty bit tracking.");
1848	PGM_REG_COUNTER(&pPgmCpu->StatRZDirtyPageSkipped, "/PGM/CPU%u/RZ/DirtyPage/Skipped", "The number of pages already dirty or readonly.");
1849	PGM_REG_COUNTER(&pPgmCpu->StatRZDirtyPageTrap, "/PGM/CPU%u/RZ/DirtyPage/Trap", "The number of traps generated for dirty bit tracking.");
1850	PGM_REG_COUNTER(&pPgmCpu->StatRZDirtyPageStale, "/PGM/CPU%u/RZ/DirtyPage/Stale", "The number of traps generated for dirty bit tracking (stale tlb entries).");
1851	PGM_REG_COUNTER(&pPgmCpu->StatRZDirtiedPage, "/PGM/CPU%u/RZ/DirtyPage/SetDirty", "The number of pages marked dirty because of write accesses.");
1852	PGM_REG_COUNTER(&pPgmCpu->StatRZDirtyTrackRealPF, "/PGM/CPU%u/RZ/DirtyPage/RealPF", "The number of real pages faults during dirty bit tracking.");
1853	PGM_REG_COUNTER(&pPgmCpu->StatRZPageAlreadyDirty, "/PGM/CPU%u/RZ/DirtyPage/AlreadySet", "The number of pages already marked dirty because of write accesses.");
1854	PGM_REG_PROFILE(&pPgmCpu->StatRZInvalidatePage, "/PGM/CPU%u/RZ/InvalidatePage", "PGMInvalidatePage() profiling.");
1855	PGM_REG_COUNTER(&pPgmCpu->StatRZInvalidatePage4KBPages, "/PGM/CPU%u/RZ/InvalidatePage/4KBPages", "The number of times PGMInvalidatePage() was called for a 4KB page.");
1856	PGM_REG_COUNTER(&pPgmCpu->StatRZInvalidatePage4MBPages, "/PGM/CPU%u/RZ/InvalidatePage/4MBPages", "The number of times PGMInvalidatePage() was called for a 4MB page.");
1857	PGM_REG_COUNTER(&pPgmCpu->StatRZInvalidatePage4MBPagesSkip, "/PGM/CPU%u/RZ/InvalidatePage/4MBPagesSkip","The number of times PGMInvalidatePage() skipped a 4MB page.");
1858	PGM_REG_COUNTER(&pPgmCpu->StatRZInvalidatePagePDMappings, "/PGM/CPU%u/RZ/InvalidatePage/PDMappings", "The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict).");
1859	PGM_REG_COUNTER(&pPgmCpu->StatRZInvalidatePagePDNAs, "/PGM/CPU%u/RZ/InvalidatePage/PDNAs", "The number of times PGMInvalidatePage() was called for a not accessed page directory.");
1860	PGM_REG_COUNTER(&pPgmCpu->StatRZInvalidatePagePDNPs, "/PGM/CPU%u/RZ/InvalidatePage/PDNPs", "The number of times PGMInvalidatePage() was called for a not present page directory.");
1861	PGM_REG_COUNTER(&pPgmCpu->StatRZInvalidatePagePDOutOfSync, "/PGM/CPU%u/RZ/InvalidatePage/PDOutOfSync", "The number of times PGMInvalidatePage() was called for an out of sync page directory.");
1862	PGM_REG_COUNTER(&pPgmCpu->StatRZInvalidatePageSkipped, "/PGM/CPU%u/RZ/InvalidatePage/Skipped", "The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3.");
1863	PGM_REG_COUNTER(&pPgmCpu->StatRZPageOutOfSyncSupervisor, "/PGM/CPU%u/RZ/OutOfSync/SuperVisor", "Number of traps due to pages out of sync (P) and times VerifyAccessSyncPage calls SyncPage.");
1864	PGM_REG_COUNTER(&pPgmCpu->StatRZPageOutOfSyncUser, "/PGM/CPU%u/RZ/OutOfSync/User", "Number of traps due to pages out of sync (P) and times VerifyAccessSyncPage calls SyncPage.");
1865	PGM_REG_COUNTER(&pPgmCpu->StatRZPageOutOfSyncSupervisorWrite,"/PGM/CPU%u/RZ/OutOfSync/SuperVisorWrite", "Number of traps due to pages out of sync (RW) and times VerifyAccessSyncPage calls SyncPage.");
1866	PGM_REG_COUNTER(&pPgmCpu->StatRZPageOutOfSyncUserWrite, "/PGM/CPU%u/RZ/OutOfSync/UserWrite", "Number of traps due to pages out of sync (RW) and times VerifyAccessSyncPage calls SyncPage.");
1867	PGM_REG_PROFILE(&pPgmCpu->StatRZPrefetch, "/PGM/CPU%u/RZ/Prefetch", "PGMPrefetchPage profiling.");
1868	PGM_REG_PROFILE(&pPgmCpu->StatRZFlushTLB, "/PGM/CPU%u/RZ/FlushTLB", "Profiling of the PGMFlushTLB() body.");
1869	PGM_REG_COUNTER(&pPgmCpu->StatRZFlushTLBNewCR3, "/PGM/CPU%u/RZ/FlushTLB/NewCR3", "The number of times PGMFlushTLB was called with a new CR3, non-global. (switch)");
1870	PGM_REG_COUNTER(&pPgmCpu->StatRZFlushTLBNewCR3Global, "/PGM/CPU%u/RZ/FlushTLB/NewCR3Global", "The number of times PGMFlushTLB was called with a new CR3, global. (switch)");
1871	PGM_REG_COUNTER(&pPgmCpu->StatRZFlushTLBSameCR3, "/PGM/CPU%u/RZ/FlushTLB/SameCR3", "The number of times PGMFlushTLB was called with the same CR3, non-global. (flush)");
1872	PGM_REG_COUNTER(&pPgmCpu->StatRZFlushTLBSameCR3Global, "/PGM/CPU%u/RZ/FlushTLB/SameCR3Global", "The number of times PGMFlushTLB was called with the same CR3, global. (flush)");
1873	PGM_REG_PROFILE(&pPgmCpu->StatRZGstModifyPage, "/PGM/CPU%u/RZ/GstModifyPage", "Profiling of the PGMGstModifyPage() body.");
1874
1875	PGM_REG_PROFILE(&pPgmCpu->StatR3SyncCR3, "/PGM/CPU%u/R3/SyncCR3", "Profiling of the PGMSyncCR3() body.");
1876	PGM_REG_PROFILE(&pPgmCpu->StatR3SyncCR3Handlers, "/PGM/CPU%u/R3/SyncCR3/Handlers", "Profiling of the PGMSyncCR3() update handler section.");
1877	PGM_REG_COUNTER(&pPgmCpu->StatR3SyncCR3Global, "/PGM/CPU%u/R3/SyncCR3/Global", "The number of global CR3 syncs.");
1878	PGM_REG_COUNTER(&pPgmCpu->StatR3SyncCR3NotGlobal, "/PGM/CPU%u/R3/SyncCR3/NotGlobal", "The number of non-global CR3 syncs.");
1879	PGM_REG_COUNTER(&pPgmCpu->StatR3SyncCR3DstCacheHit, "/PGM/CPU%u/R3/SyncCR3/DstChacheHit", "The number of times we got some kind of a cache hit.");
1880	PGM_REG_COUNTER(&pPgmCpu->StatR3SyncCR3DstFreed, "/PGM/CPU%u/R3/SyncCR3/DstFreed", "The number of times we've had to free a shadow entry.");
1881	PGM_REG_COUNTER(&pPgmCpu->StatR3SyncCR3DstFreedSrcNP, "/PGM/CPU%u/R3/SyncCR3/DstFreedSrcNP", "The number of times we've had to free a shadow entry for which the source entry was not present.");
1882	PGM_REG_COUNTER(&pPgmCpu->StatR3SyncCR3DstNotPresent, "/PGM/CPU%u/R3/SyncCR3/DstNotPresent", "The number of times we've encountered a not present shadow entry for a present guest entry.");
1883	PGM_REG_COUNTER(&pPgmCpu->StatR3SyncCR3DstSkippedGlobalPD, "/PGM/CPU%u/R3/SyncCR3/DstSkippedGlobalPD", "The number of times a global page directory wasn't flushed.");
1884	PGM_REG_COUNTER(&pPgmCpu->StatR3SyncCR3DstSkippedGlobalPT, "/PGM/CPU%u/R3/SyncCR3/DstSkippedGlobalPT", "The number of times a page table with only global entries wasn't flushed.");
1885	PGM_REG_PROFILE(&pPgmCpu->StatR3SyncPT, "/PGM/CPU%u/R3/SyncPT", "Profiling of the pfnSyncPT() body.");
1886	PGM_REG_COUNTER(&pPgmCpu->StatR3SyncPTFailed, "/PGM/CPU%u/R3/SyncPT/Failed", "The number of times pfnSyncPT() failed.");
1887	PGM_REG_COUNTER(&pPgmCpu->StatR3SyncPT4K, "/PGM/CPU%u/R3/SyncPT/4K", "Nr of 4K PT syncs");
1888	PGM_REG_COUNTER(&pPgmCpu->StatR3SyncPT4M, "/PGM/CPU%u/R3/SyncPT/4M", "Nr of 4M PT syncs");
1889	PGM_REG_COUNTER(&pPgmCpu->StatR3SyncPagePDNAs, "/PGM/CPU%u/R3/SyncPagePDNAs", "The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit.");
1890	PGM_REG_COUNTER(&pPgmCpu->StatR3SyncPagePDOutOfSync, "/PGM/CPU%u/R3/SyncPagePDOutOfSync", "The number of time we've encountered an out-of-sync PD in SyncPage.");
1891	PGM_REG_COUNTER(&pPgmCpu->StatR3AccessedPage, "/PGM/CPU%u/R3/AccessedPage", "The number of pages marked not present for accessed bit emulation.");
1892	PGM_REG_PROFILE(&pPgmCpu->StatR3DirtyBitTracking, "/PGM/CPU%u/R3/DirtyPage", "Profiling the dirty bit tracking in CheckPageFault().");
1893	PGM_REG_COUNTER(&pPgmCpu->StatR3DirtyPage, "/PGM/CPU%u/R3/DirtyPage/Mark", "The number of pages marked read-only for dirty bit tracking.");
1894	PGM_REG_COUNTER(&pPgmCpu->StatR3DirtyPageBig, "/PGM/CPU%u/R3/DirtyPage/MarkBig", "The number of 4MB pages marked read-only for dirty bit tracking.");
1895	PGM_REG_COUNTER(&pPgmCpu->StatR3DirtyPageSkipped, "/PGM/CPU%u/R3/DirtyPage/Skipped", "The number of pages already dirty or readonly.");
1896	PGM_REG_COUNTER(&pPgmCpu->StatR3DirtyPageTrap, "/PGM/CPU%u/R3/DirtyPage/Trap", "The number of traps generated for dirty bit tracking.");
1897	PGM_REG_COUNTER(&pPgmCpu->StatR3DirtiedPage, "/PGM/CPU%u/R3/DirtyPage/SetDirty", "The number of pages marked dirty because of write accesses.");
1898	PGM_REG_COUNTER(&pPgmCpu->StatR3DirtyTrackRealPF, "/PGM/CPU%u/R3/DirtyPage/RealPF", "The number of real pages faults during dirty bit tracking.");
1899	PGM_REG_COUNTER(&pPgmCpu->StatR3PageAlreadyDirty, "/PGM/CPU%u/R3/DirtyPage/AlreadySet", "The number of pages already marked dirty because of write accesses.");
1900	PGM_REG_PROFILE(&pPgmCpu->StatR3InvalidatePage, "/PGM/CPU%u/R3/InvalidatePage", "PGMInvalidatePage() profiling.");
1901	PGM_REG_COUNTER(&pPgmCpu->StatR3InvalidatePage4KBPages, "/PGM/CPU%u/R3/InvalidatePage/4KBPages", "The number of times PGMInvalidatePage() was called for a 4KB page.");
1902	PGM_REG_COUNTER(&pPgmCpu->StatR3InvalidatePage4MBPages, "/PGM/CPU%u/R3/InvalidatePage/4MBPages", "The number of times PGMInvalidatePage() was called for a 4MB page.");
1903	PGM_REG_COUNTER(&pPgmCpu->StatR3InvalidatePage4MBPagesSkip, "/PGM/CPU%u/R3/InvalidatePage/4MBPagesSkip","The number of times PGMInvalidatePage() skipped a 4MB page.");
1904	PGM_REG_COUNTER(&pPgmCpu->StatR3InvalidatePagePDMappings, "/PGM/CPU%u/R3/InvalidatePage/PDMappings", "The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict).");
1905	PGM_REG_COUNTER(&pPgmCpu->StatR3InvalidatePagePDNAs, "/PGM/CPU%u/R3/InvalidatePage/PDNAs", "The number of times PGMInvalidatePage() was called for a not accessed page directory.");
1906	PGM_REG_COUNTER(&pPgmCpu->StatR3InvalidatePagePDNPs, "/PGM/CPU%u/R3/InvalidatePage/PDNPs", "The number of times PGMInvalidatePage() was called for a not present page directory.");
1907	PGM_REG_COUNTER(&pPgmCpu->StatR3InvalidatePagePDOutOfSync, "/PGM/CPU%u/R3/InvalidatePage/PDOutOfSync", "The number of times PGMInvalidatePage() was called for an out of sync page directory.");
1908	PGM_REG_COUNTER(&pPgmCpu->StatR3InvalidatePageSkipped, "/PGM/CPU%u/R3/InvalidatePage/Skipped", "The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3.");
1909	PGM_REG_COUNTER(&pPgmCpu->StatR3PageOutOfSyncSupervisor, "/PGM/CPU%u/R3/OutOfSync/SuperVisor", "Number of traps due to pages out of sync and times VerifyAccessSyncPage calls SyncPage.");
1910	PGM_REG_COUNTER(&pPgmCpu->StatR3PageOutOfSyncUser, "/PGM/CPU%u/R3/OutOfSync/User", "Number of traps due to pages out of sync and times VerifyAccessSyncPage calls SyncPage.");
1911	PGM_REG_PROFILE(&pPgmCpu->StatR3Prefetch, "/PGM/CPU%u/R3/Prefetch", "PGMPrefetchPage profiling.");
1912	PGM_REG_PROFILE(&pPgmCpu->StatR3FlushTLB, "/PGM/CPU%u/R3/FlushTLB", "Profiling of the PGMFlushTLB() body.");
1913	PGM_REG_COUNTER(&pPgmCpu->StatR3FlushTLBNewCR3, "/PGM/CPU%u/R3/FlushTLB/NewCR3", "The number of times PGMFlushTLB was called with a new CR3, non-global. (switch)");
1914	PGM_REG_COUNTER(&pPgmCpu->StatR3FlushTLBNewCR3Global, "/PGM/CPU%u/R3/FlushTLB/NewCR3Global", "The number of times PGMFlushTLB was called with a new CR3, global. (switch)");
1915	PGM_REG_COUNTER(&pPgmCpu->StatR3FlushTLBSameCR3, "/PGM/CPU%u/R3/FlushTLB/SameCR3", "The number of times PGMFlushTLB was called with the same CR3, non-global. (flush)");
1916	PGM_REG_COUNTER(&pPgmCpu->StatR3FlushTLBSameCR3Global, "/PGM/CPU%u/R3/FlushTLB/SameCR3Global", "The number of times PGMFlushTLB was called with the same CR3, global. (flush)");
1917	PGM_REG_PROFILE(&pPgmCpu->StatR3GstModifyPage, "/PGM/CPU%u/R3/GstModifyPage", "Profiling of the PGMGstModifyPage() body.");
1918	#endif /* VBOX_WITH_STATISTICS */
1919
1920	#undef PGM_REG_PROFILE
1921	#undef PGM_REG_COUNTER
1922
1923	}
1924	}
1925
1926
1927	/**
1928	* Init the PGM bits that rely on VMMR0 and MM to be fully initialized.
1929	*
1930	* The dynamic mapping area will also be allocated and initialized at this
1931	* time. We could allocate it during PGMR3Init of course, but the mapping
1932	* wouldn't be allocated at that time preventing us from setting up the
1933	* page table entries with the dummy page.
1934	*
1935	* @returns VBox status code.
1936	* @param pVM VM handle.
1937	*/
1938	VMMR3DECL(int) PGMR3InitDynMap(PVM pVM)
1939	{
1940	RTGCPTR GCPtr;
1941	int rc;
1942
1943	/*
1944	* Reserve space for the dynamic mappings.
1945	*/
1946	rc = MMR3HyperReserve(pVM, MM_HYPER_DYNAMIC_SIZE, "Dynamic mapping", &GCPtr);
1947	if (RT_SUCCESS(rc))
1948	pVM->pgm.s.pbDynPageMapBaseGC = GCPtr;
1949
1950	if ( RT_SUCCESS(rc)
1951	&& (pVM->pgm.s.pbDynPageMapBaseGC >> X86_PD_PAE_SHIFT) != ((pVM->pgm.s.pbDynPageMapBaseGC + MM_HYPER_DYNAMIC_SIZE - 1) >> X86_PD_PAE_SHIFT))
1952	{
1953	rc = MMR3HyperReserve(pVM, MM_HYPER_DYNAMIC_SIZE, "Dynamic mapping not crossing", &GCPtr);
1954	if (RT_SUCCESS(rc))
1955	pVM->pgm.s.pbDynPageMapBaseGC = GCPtr;
1956	}
1957	if (RT_SUCCESS(rc))
1958	{
1959	AssertRelease((pVM->pgm.s.pbDynPageMapBaseGC >> X86_PD_PAE_SHIFT) == ((pVM->pgm.s.pbDynPageMapBaseGC + MM_HYPER_DYNAMIC_SIZE - 1) >> X86_PD_PAE_SHIFT));
1960	MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
1961	}
1962	return rc;
1963	}
1964
1965
1966	/**
1967	* Ring-3 init finalizing.
1968	*
1969	* @returns VBox status code.
1970	* @param pVM The VM handle.
1971	*/
1972	VMMR3DECL(int) PGMR3InitFinalize(PVM pVM)
1973	{
1974	int rc;
1975
1976	/*
1977	* Reserve space for the dynamic mappings.
1978	* Initialize the dynamic mapping pages with dummy pages to simply the cache.
1979	*/
1980	/* get the pointer to the page table entries. */
1981	PPGMMAPPING pMapping = pgmGetMapping(pVM, pVM->pgm.s.pbDynPageMapBaseGC);
1982	AssertRelease(pMapping);
1983	const uintptr_t off = pVM->pgm.s.pbDynPageMapBaseGC - pMapping->GCPtr;
1984	const unsigned iPT = off >> X86_PD_SHIFT;
1985	const unsigned iPG = (off >> X86_PT_SHIFT) & X86_PT_MASK;
1986	pVM->pgm.s.paDynPageMap32BitPTEsGC = pMapping->aPTs[iPT].pPTRC + iPG * sizeof(pMapping->aPTs[0].pPTR3->a[0]);
1987	pVM->pgm.s.paDynPageMapPaePTEsGC = pMapping->aPTs[iPT].paPaePTsRC + iPG * sizeof(pMapping->aPTs[0].paPaePTsR3->a[0]);
1988
1989	/* init cache */
1990	RTHCPHYS HCPhysDummy = MMR3PageDummyHCPhys(pVM);
1991	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHCPhysDynPageMapCache); i++)
1992	pVM->pgm.s.aHCPhysDynPageMapCache[i] = HCPhysDummy;
1993
1994	for (unsigned i = 0; i < MM_HYPER_DYNAMIC_SIZE; i += PAGE_SIZE)
1995	{
1996	rc = PGMMap(pVM, pVM->pgm.s.pbDynPageMapBaseGC + i, HCPhysDummy, PAGE_SIZE, 0);
1997	AssertRCReturn(rc, rc);
1998	}
1999
2000	/*
2001	* Note that AMD uses all the 8 reserved bits for the address (so 40 bits in total);
2002	* Intel only goes up to 36 bits, so we stick to 36 as well.
2003	*/
2004	/** @todo How to test for the 40 bits support? Long mode seems to be the test criterium. */
2005	uint32_t u32Dummy, u32Features;
2006	CPUMGetGuestCpuId(VMMGetCpu(pVM), 1, &u32Dummy, &u32Dummy, &u32Dummy, &u32Features);
2007
2008	if (u32Features & X86_CPUID_FEATURE_EDX_PSE36)
2009	pVM->pgm.s.GCPhys4MBPSEMask = RT_BIT_64(36) - 1;
2010	else
2011	pVM->pgm.s.GCPhys4MBPSEMask = RT_BIT_64(32) - 1;
2012
2013	/*
2014	* Allocate memory if we're supposed to do that.
2015	*/
2016	if (pVM->pgm.s.fRamPreAlloc)
2017	rc = pgmR3PhysRamPreAllocate(pVM);
2018
2019	LogRel(("PGMR3InitFinalize: 4 MB PSE mask %RGp\n", pVM->pgm.s.GCPhys4MBPSEMask));
2020	return rc;
2021	}
2022
2023
2024	/**
2025	* Applies relocations to data and code managed by this component.
2026	*
2027	* This function will be called at init and whenever the VMM need to relocate it
2028	* self inside the GC.
2029	*
2030	* @param pVM The VM.
2031	* @param offDelta Relocation delta relative to old location.
2032	*/
2033	VMMR3DECL(void) PGMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
2034	{
2035	LogFlow(("PGMR3Relocate %RGv to %RGv\n", pVM->pgm.s.GCPtrCR3Mapping, pVM->pgm.s.GCPtrCR3Mapping + offDelta));
2036
2037	/*
2038	* Paging stuff.
2039	*/
2040	pVM->pgm.s.GCPtrCR3Mapping += offDelta;
2041
2042	pgmR3ModeDataInit(pVM, true /* resolve GC/R0 symbols */);
2043
2044	/* Shadow, guest and both mode switch & relocation for each VCPU. */
2045	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2046	{
2047	PVMCPU pVCpu = &pVM->aCpus[i];
2048
2049	pgmR3ModeDataSwitch(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
2050
2051	PGM_SHW_PFN(Relocate, pVCpu)(pVCpu, offDelta);
2052	PGM_GST_PFN(Relocate, pVCpu)(pVCpu, offDelta);
2053	PGM_BTH_PFN(Relocate, pVCpu)(pVCpu, offDelta);
2054	}
2055
2056	/*
2057	* Trees.
2058	*/
2059	pVM->pgm.s.pTreesRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pTreesR3);
2060
2061	/*
2062	* Ram ranges.
2063	*/
2064	if (pVM->pgm.s.pRamRangesR3)
2065	{
2066	/* Update the pSelfRC pointers and relink them. */
2067	for (PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
2068	if (!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING))
2069	pCur->pSelfRC = MMHyperCCToRC(pVM, pCur);
2070	pgmR3PhysRelinkRamRanges(pVM);
2071	}
2072
2073	/*
2074	* Update the pSelfRC pointer of the MMIO2 ram ranges since they might not
2075	* be mapped and thus not included in the above exercise.
2076	*/
2077	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2078	if (!(pCur->RamRange.fFlags & PGM_RAM_RANGE_FLAGS_FLOATING))
2079	pCur->RamRange.pSelfRC = MMHyperCCToRC(pVM, &pCur->RamRange);
2080
2081	/*
2082	* Update the two page directories with all page table mappings.
2083	* (One or more of them have changed, that's why we're here.)
2084	*/
2085	pVM->pgm.s.pMappingsRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pMappingsR3);
2086	for (PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3; pCur->pNextR3; pCur = pCur->pNextR3)
2087	pCur->pNextRC = MMHyperR3ToRC(pVM, pCur->pNextR3);
2088
2089	/* Relocate GC addresses of Page Tables. */
2090	for (PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3; pCur; pCur = pCur->pNextR3)
2091	{
2092	for (RTHCUINT i = 0; i < pCur->cPTs; i++)
2093	{
2094	pCur->aPTs[i].pPTRC = MMHyperR3ToRC(pVM, pCur->aPTs[i].pPTR3);
2095	pCur->aPTs[i].paPaePTsRC = MMHyperR3ToRC(pVM, pCur->aPTs[i].paPaePTsR3);
2096	}
2097	}
2098
2099	/*
2100	* Dynamic page mapping area.
2101	*/
2102	pVM->pgm.s.paDynPageMap32BitPTEsGC += offDelta;
2103	pVM->pgm.s.paDynPageMapPaePTEsGC += offDelta;
2104	pVM->pgm.s.pbDynPageMapBaseGC += offDelta;
2105
2106	/*
2107	* The Zero page.
2108	*/
2109	pVM->pgm.s.pvZeroPgR0 = MMHyperR3ToR0(pVM, pVM->pgm.s.pvZeroPgR3);
2110	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2111	AssertRelease(pVM->pgm.s.pvZeroPgR0 != NIL_RTR0PTR \|\| !VMMIsHwVirtExtForced(pVM));
2112	#else
2113	AssertRelease(pVM->pgm.s.pvZeroPgR0 != NIL_RTR0PTR);
2114	#endif
2115
2116	/*
2117	* Physical and virtual handlers.
2118	*/
2119	RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers, true, pgmR3RelocatePhysHandler, &offDelta);
2120	RTAvlroGCPtrDoWithAll(&pVM->pgm.s.pTreesR3->VirtHandlers, true, pgmR3RelocateVirtHandler, &offDelta);
2121	RTAvlroGCPtrDoWithAll(&pVM->pgm.s.pTreesR3->HyperVirtHandlers, true, pgmR3RelocateHyperVirtHandler, &offDelta);
2122
2123	/*
2124	* The page pool.
2125	*/
2126	pgmR3PoolRelocate(pVM);
2127	}
2128
2129
2130	/**
2131	* Callback function for relocating a physical access handler.
2132	*
2133	* @returns 0 (continue enum)
2134	* @param pNode Pointer to a PGMPHYSHANDLER node.
2135	* @param pvUser Pointer to the offDelta. This is a pointer to the delta since we're
2136	* not certain the delta will fit in a void pointer for all possible configs.
2137	*/
2138	static DECLCALLBACK(int) pgmR3RelocatePhysHandler(PAVLROGCPHYSNODECORE pNode, void *pvUser)
2139	{
2140	PPGMPHYSHANDLER pHandler = (PPGMPHYSHANDLER)pNode;
2141	RTGCINTPTR offDelta = *(PRTGCINTPTR)pvUser;
2142	if (pHandler->pfnHandlerRC)
2143	pHandler->pfnHandlerRC += offDelta;
2144	if (pHandler->pvUserRC >= 0x10000)
2145	pHandler->pvUserRC += offDelta;
2146	return 0;
2147	}
2148
2149
2150	/**
2151	* Callback function for relocating a virtual access handler.
2152	*
2153	* @returns 0 (continue enum)
2154	* @param pNode Pointer to a PGMVIRTHANDLER node.
2155	* @param pvUser Pointer to the offDelta. This is a pointer to the delta since we're
2156	* not certain the delta will fit in a void pointer for all possible configs.
2157	*/
2158	static DECLCALLBACK(int) pgmR3RelocateVirtHandler(PAVLROGCPTRNODECORE pNode, void *pvUser)
2159	{
2160	PPGMVIRTHANDLER pHandler = (PPGMVIRTHANDLER)pNode;
2161	RTGCINTPTR offDelta = *(PRTGCINTPTR)pvUser;
2162	Assert( pHandler->enmType == PGMVIRTHANDLERTYPE_ALL
2163	\|\| pHandler->enmType == PGMVIRTHANDLERTYPE_WRITE);
2164	Assert(pHandler->pfnHandlerRC);
2165	pHandler->pfnHandlerRC += offDelta;
2166	return 0;
2167	}
2168
2169
2170	/**
2171	* Callback function for relocating a virtual access handler for the hypervisor mapping.
2172	*
2173	* @returns 0 (continue enum)
2174	* @param pNode Pointer to a PGMVIRTHANDLER node.
2175	* @param pvUser Pointer to the offDelta. This is a pointer to the delta since we're
2176	* not certain the delta will fit in a void pointer for all possible configs.
2177	*/
2178	static DECLCALLBACK(int) pgmR3RelocateHyperVirtHandler(PAVLROGCPTRNODECORE pNode, void *pvUser)
2179	{
2180	PPGMVIRTHANDLER pHandler = (PPGMVIRTHANDLER)pNode;
2181	RTGCINTPTR offDelta = *(PRTGCINTPTR)pvUser;
2182	Assert(pHandler->enmType == PGMVIRTHANDLERTYPE_HYPERVISOR);
2183	Assert(pHandler->pfnHandlerRC);
2184	pHandler->pfnHandlerRC += offDelta;
2185	return 0;
2186	}
2187
2188
2189	/**
2190	* Resets a virtual CPU when unplugged.
2191	*
2192	* @param pVM The VM handle.
2193	* @param pVCpu The virtual CPU handle.
2194	*/
2195	VMMR3DECL(void) PGMR3ResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu)
2196	{
2197	int rc = PGM_GST_PFN(Exit, pVCpu)(pVCpu);
2198	AssertRC(rc);
2199
2200	rc = PGMR3ChangeMode(pVM, pVCpu, PGMMODE_REAL);
2201	AssertRC(rc);
2202
2203	STAM_REL_COUNTER_RESET(&pVCpu->pgm.s.cGuestModeChanges);
2204
2205	pgmR3PoolResetUnpluggedCpu(pVM, pVCpu);
2206
2207	/*
2208	* Re-init other members.
2209	*/
2210	pVCpu->pgm.s.fA20Enabled = true;
2211
2212	/*
2213	* Clear the FFs PGM owns.
2214	*/
2215	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2216	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2217	}
2218
2219
2220	/**
2221	* The VM is being reset.
2222	*
2223	* For the PGM component this means that any PD write monitors
2224	* needs to be removed.
2225	*
2226	* @param pVM VM handle.
2227	*/
2228	VMMR3DECL(void) PGMR3Reset(PVM pVM)
2229	{
2230	int rc;
2231
2232	LogFlow(("PGMR3Reset:\n"));
2233	VM_ASSERT_EMT(pVM);
2234
2235	pgmLock(pVM);
2236
2237	/*
2238	* Unfix any fixed mappings and disable CR3 monitoring.
2239	*/
2240	pVM->pgm.s.fMappingsFixed = false;
2241	pVM->pgm.s.fMappingsFixedRestored = false;
2242	pVM->pgm.s.GCPtrMappingFixed = NIL_RTGCPTR;
2243	pVM->pgm.s.cbMappingFixed = 0;
2244
2245	/*
2246	* Exit the guest paging mode before the pgm pool gets reset.
2247	* Important to clean up the amd64 case.
2248	*/
2249	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2250	{
2251	PVMCPU pVCpu = &pVM->aCpus[i];
2252	rc = PGM_GST_PFN(Exit, pVCpu)(pVCpu);
2253	AssertRC(rc);
2254	}
2255
2256	#ifdef DEBUG
2257	DBGFR3InfoLog(pVM, "mappings", NULL);
2258	DBGFR3InfoLog(pVM, "handlers", "all nostat");
2259	#endif
2260
2261	/*
2262	* Switch mode back to real mode. (before resetting the pgm pool!)
2263	*/
2264	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2265	{
2266	PVMCPU pVCpu = &pVM->aCpus[i];
2267
2268	rc = PGMR3ChangeMode(pVM, pVCpu, PGMMODE_REAL);
2269	AssertRC(rc);
2270
2271	STAM_REL_COUNTER_RESET(&pVCpu->pgm.s.cGuestModeChanges);
2272	}
2273
2274	/*
2275	* Reset the shadow page pool.
2276	*/
2277	pgmR3PoolReset(pVM);
2278
2279	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2280	{
2281	PVMCPU pVCpu = &pVM->aCpus[i];
2282
2283	/*
2284	* Re-init other members.
2285	*/
2286	pVCpu->pgm.s.fA20Enabled = true;
2287
2288	/*
2289	* Clear the FFs PGM owns.
2290	*/
2291	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2292	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2293	}
2294
2295	/*
2296	* Reset (zero) RAM pages.
2297	*/
2298	rc = pgmR3PhysRamReset(pVM);
2299	if (RT_SUCCESS(rc))
2300	{
2301	/*
2302	* Reset (zero) shadow ROM pages.
2303	*/
2304	rc = pgmR3PhysRomReset(pVM);
2305	}
2306
2307	pgmUnlock(pVM);
2308	AssertReleaseRC(rc);
2309	}
2310
2311
2312	#ifdef VBOX_STRICT
2313	/**
2314	* VM state change callback for clearing fNoMorePhysWrites after
2315	* a snapshot has been created.
2316	*/
2317	static DECLCALLBACK(void) pgmR3ResetNoMorePhysWritesFlag(PVM pVM, VMSTATE enmState, VMSTATE enmOldState, void *pvUser)
2318	{
2319	if ( enmState == VMSTATE_RUNNING
2320	\|\| enmState == VMSTATE_RESUMING)
2321	pVM->pgm.s.fNoMorePhysWrites = false;
2322	}
2323	#endif
2324
2325
2326	/**
2327	* Terminates the PGM.
2328	*
2329	* @returns VBox status code.
2330	* @param pVM Pointer to VM structure.
2331	*/
2332	VMMR3DECL(int) PGMR3Term(PVM pVM)
2333	{
2334	PGMDeregisterStringFormatTypes();
2335	return PDMR3CritSectDelete(&pVM->pgm.s.CritSect);
2336	}
2337
2338
2339	/**
2340	* Terminates the per-VCPU PGM.
2341	*
2342	* Termination means cleaning up and freeing all resources,
2343	* the VM it self is at this point powered off or suspended.
2344	*
2345	* @returns VBox status code.
2346	* @param pVM The VM to operate on.
2347	*/
2348	VMMR3DECL(int) PGMR3TermCPU(PVM pVM)
2349	{
2350	return 0;
2351	}
2352
2353
2354	/**
2355	* Show paging mode.
2356	*
2357	* @param pVM VM Handle.
2358	* @param pHlp The info helpers.
2359	* @param pszArgs "all" (default), "guest", "shadow" or "host".
2360	*/
2361	static DECLCALLBACK(void) pgmR3InfoMode(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
2362	{
2363	/* digest argument. */
2364	bool fGuest, fShadow, fHost;
2365	if (pszArgs)
2366	pszArgs = RTStrStripL(pszArgs);
2367	if (!pszArgs \|\| !*pszArgs \|\| strstr(pszArgs, "all"))
2368	fShadow = fHost = fGuest = true;
2369	else
2370	{
2371	fShadow = fHost = fGuest = false;
2372	if (strstr(pszArgs, "guest"))
2373	fGuest = true;
2374	if (strstr(pszArgs, "shadow"))
2375	fShadow = true;
2376	if (strstr(pszArgs, "host"))
2377	fHost = true;
2378	}
2379
2380	/** @todo SMP support! */
2381	/* print info. */
2382	if (fGuest)
2383	pHlp->pfnPrintf(pHlp, "Guest paging mode: %s, changed %RU64 times, A20 %s\n",
2384	PGMGetModeName(pVM->aCpus[0].pgm.s.enmGuestMode), pVM->aCpus[0].pgm.s.cGuestModeChanges.c,
2385	pVM->aCpus[0].pgm.s.fA20Enabled ? "enabled" : "disabled");
2386	if (fShadow)
2387	pHlp->pfnPrintf(pHlp, "Shadow paging mode: %s\n", PGMGetModeName(pVM->aCpus[0].pgm.s.enmShadowMode));
2388	if (fHost)
2389	{
2390	const char *psz;
2391	switch (pVM->pgm.s.enmHostMode)
2392	{
2393	case SUPPAGINGMODE_INVALID: psz = "invalid"; break;
2394	case SUPPAGINGMODE_32_BIT: psz = "32-bit"; break;
2395	case SUPPAGINGMODE_32_BIT_GLOBAL: psz = "32-bit+G"; break;
2396	case SUPPAGINGMODE_PAE: psz = "PAE"; break;
2397	case SUPPAGINGMODE_PAE_GLOBAL: psz = "PAE+G"; break;
2398	case SUPPAGINGMODE_PAE_NX: psz = "PAE+NX"; break;
2399	case SUPPAGINGMODE_PAE_GLOBAL_NX: psz = "PAE+G+NX"; break;
2400	case SUPPAGINGMODE_AMD64: psz = "AMD64"; break;
2401	case SUPPAGINGMODE_AMD64_GLOBAL: psz = "AMD64+G"; break;
2402	case SUPPAGINGMODE_AMD64_NX: psz = "AMD64+NX"; break;
2403	case SUPPAGINGMODE_AMD64_GLOBAL_NX: psz = "AMD64+G+NX"; break;
2404	default: psz = "unknown"; break;
2405	}
2406	pHlp->pfnPrintf(pHlp, "Host paging mode: %s\n", psz);
2407	}
2408	}
2409
2410
2411	/**
2412	* Dump registered MMIO ranges to the log.
2413	*
2414	* @param pVM VM Handle.
2415	* @param pHlp The info helpers.
2416	* @param pszArgs Arguments, ignored.
2417	*/
2418	static DECLCALLBACK(void) pgmR3PhysInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
2419	{
2420	NOREF(pszArgs);
2421	pHlp->pfnPrintf(pHlp,
2422	"RAM ranges (pVM=%p)\n"
2423	"%.s %.s\n",
2424	pVM,
2425	sizeof(RTGCPHYS) * 4 + 1, "GC Phys Range ",
2426	sizeof(RTHCPTR) * 2, "pvHC ");
2427
2428	for (PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
2429	pHlp->pfnPrintf(pHlp,
2430	"%RGp-%RGp %RHv %s\n",
2431	pCur->GCPhys,
2432	pCur->GCPhysLast,
2433	pCur->pvR3,
2434	pCur->pszDesc);
2435	}
2436
2437	/**
2438	* Dump the page directory to the log.
2439	*
2440	* @param pVM VM Handle.
2441	* @param pHlp The info helpers.
2442	* @param pszArgs Arguments, ignored.
2443	*/
2444	static DECLCALLBACK(void) pgmR3InfoCr3(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
2445	{
2446	/** @todo SMP support!! */
2447	PVMCPU pVCpu = &pVM->aCpus[0];
2448
2449	/** @todo fix this! Convert the PGMR3DumpHierarchyHC functions to do guest stuff. */
2450	/* Big pages supported? */
2451	const bool fPSE = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
2452
2453	/* Global pages supported? */
2454	const bool fPGE = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PGE);
2455
2456	NOREF(pszArgs);
2457
2458	/*
2459	* Get page directory addresses.
2460	*/
2461	PX86PD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
2462	Assert(pPDSrc);
2463	Assert(PGMPhysGCPhys2R3PtrAssert(pVM, (RTGCPHYS)(CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK), sizeof(*pPDSrc)) == pPDSrc);
2464
2465	/*
2466	* Iterate the page directory.
2467	*/
2468	for (unsigned iPD = 0; iPD < RT_ELEMENTS(pPDSrc->a); iPD++)
2469	{
2470	X86PDE PdeSrc = pPDSrc->a[iPD];
2471	if (PdeSrc.n.u1Present)
2472	{
2473	if (PdeSrc.b.u1Size && fPSE)
2474	pHlp->pfnPrintf(pHlp,
2475	"%04X - %RGp P=%d U=%d RW=%d G=%d - BIG\n",
2476	iPD,
2477	pgmGstGet4MBPhysPage(&pVM->pgm.s, PdeSrc),
2478	PdeSrc.b.u1Present, PdeSrc.b.u1User, PdeSrc.b.u1Write, PdeSrc.b.u1Global && fPGE);
2479	else
2480	pHlp->pfnPrintf(pHlp,
2481	"%04X - %RGp P=%d U=%d RW=%d [G=%d]\n",
2482	iPD,
2483	(RTGCPHYS)(PdeSrc.u & X86_PDE_PG_MASK),
2484	PdeSrc.n.u1Present, PdeSrc.n.u1User, PdeSrc.n.u1Write, PdeSrc.b.u1Global && fPGE);
2485	}
2486	}
2487	}
2488
2489
2490	/**
2491	* Service a VMMCALLRING3_PGM_LOCK call.
2492	*
2493	* @returns VBox status code.
2494	* @param pVM The VM handle.
2495	*/
2496	VMMR3DECL(int) PGMR3LockCall(PVM pVM)
2497	{
2498	int rc = PDMR3CritSectEnterEx(&pVM->pgm.s.CritSect, true /* fHostCall */);
2499	AssertRC(rc);
2500	return rc;
2501	}
2502
2503
2504	/**
2505	* Converts a PGMMODE value to a PGM_TYPE_* \#define.
2506	*
2507	* @returns PGM_TYPE_*.
2508	* @param pgmMode The mode value to convert.
2509	*/
2510	DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
2511	{
2512	switch (pgmMode)
2513	{
2514	case PGMMODE_REAL: return PGM_TYPE_REAL;
2515	case PGMMODE_PROTECTED: return PGM_TYPE_PROT;
2516	case PGMMODE_32_BIT: return PGM_TYPE_32BIT;
2517	case PGMMODE_PAE:
2518	case PGMMODE_PAE_NX: return PGM_TYPE_PAE;
2519	case PGMMODE_AMD64:
2520	case PGMMODE_AMD64_NX: return PGM_TYPE_AMD64;
2521	case PGMMODE_NESTED: return PGM_TYPE_NESTED;
2522	case PGMMODE_EPT: return PGM_TYPE_EPT;
2523	default:
2524	AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
2525	}
2526	}
2527
2528
2529	/**
2530	* Gets the index into the paging mode data array of a SHW+GST mode.
2531	*
2532	* @returns PGM::paPagingData index.
2533	* @param uShwType The shadow paging mode type.
2534	* @param uGstType The guest paging mode type.
2535	*/
2536	DECLINLINE(unsigned) pgmModeDataIndex(unsigned uShwType, unsigned uGstType)
2537	{
2538	Assert(uShwType >= PGM_TYPE_32BIT && uShwType <= PGM_TYPE_MAX);
2539	Assert(uGstType >= PGM_TYPE_REAL && uGstType <= PGM_TYPE_AMD64);
2540	return (uShwType - PGM_TYPE_32BIT) * (PGM_TYPE_AMD64 - PGM_TYPE_REAL + 1)
2541	+ (uGstType - PGM_TYPE_REAL);
2542	}
2543
2544
2545	/**
2546	* Gets the index into the paging mode data array of a SHW+GST mode.
2547	*
2548	* @returns PGM::paPagingData index.
2549	* @param enmShw The shadow paging mode.
2550	* @param enmGst The guest paging mode.
2551	*/
2552	DECLINLINE(unsigned) pgmModeDataIndexByMode(PGMMODE enmShw, PGMMODE enmGst)
2553	{
2554	Assert(enmShw >= PGMMODE_32_BIT && enmShw <= PGMMODE_MAX);
2555	Assert(enmGst > PGMMODE_INVALID && enmGst < PGMMODE_MAX);
2556	return pgmModeDataIndex(pgmModeToType(enmShw), pgmModeToType(enmGst));
2557	}
2558
2559
2560	/**
2561	* Calculates the max data index.
2562	* @returns The number of entries in the paging data array.
2563	*/
2564	DECLINLINE(unsigned) pgmModeDataMaxIndex(void)
2565	{
2566	return pgmModeDataIndex(PGM_TYPE_MAX, PGM_TYPE_AMD64) + 1;
2567	}
2568
2569
2570	/**
2571	* Initializes the paging mode data kept in PGM::paModeData.
2572	*
2573	* @param pVM The VM handle.
2574	* @param fResolveGCAndR0 Indicate whether or not GC and Ring-0 symbols can be resolved now.
2575	* This is used early in the init process to avoid trouble with PDM
2576	* not being initialized yet.
2577	*/
2578	static int pgmR3ModeDataInit(PVM pVM, bool fResolveGCAndR0)
2579	{
2580	PPGMMODEDATA pModeData;
2581	int rc;
2582
2583	/*
2584	* Allocate the array on the first call.
2585	*/
2586	if (!pVM->pgm.s.paModeData)
2587	{
2588	pVM->pgm.s.paModeData = (PPGMMODEDATA)MMR3HeapAllocZ(pVM, MM_TAG_PGM, sizeof(PGMMODEDATA) * pgmModeDataMaxIndex());
2589	AssertReturn(pVM->pgm.s.paModeData, VERR_NO_MEMORY);
2590	}
2591
2592	/*
2593	* Initialize the array entries.
2594	*/
2595	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_32BIT, PGM_TYPE_REAL)];
2596	pModeData->uShwType = PGM_TYPE_32BIT;
2597	pModeData->uGstType = PGM_TYPE_REAL;
2598	rc = PGM_SHW_NAME_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2599	rc = PGM_GST_NAME_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2600	rc = PGM_BTH_NAME_32BIT_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2601
2602	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_32BIT, PGMMODE_PROTECTED)];
2603	pModeData->uShwType = PGM_TYPE_32BIT;
2604	pModeData->uGstType = PGM_TYPE_PROT;
2605	rc = PGM_SHW_NAME_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2606	rc = PGM_GST_NAME_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2607	rc = PGM_BTH_NAME_32BIT_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2608
2609	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_32BIT, PGM_TYPE_32BIT)];
2610	pModeData->uShwType = PGM_TYPE_32BIT;
2611	pModeData->uGstType = PGM_TYPE_32BIT;
2612	rc = PGM_SHW_NAME_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2613	rc = PGM_GST_NAME_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2614	rc = PGM_BTH_NAME_32BIT_32BIT(InitData)(pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2615
2616	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_PAE, PGM_TYPE_REAL)];
2617	pModeData->uShwType = PGM_TYPE_PAE;
2618	pModeData->uGstType = PGM_TYPE_REAL;
2619	rc = PGM_SHW_NAME_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2620	rc = PGM_GST_NAME_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2621	rc = PGM_BTH_NAME_PAE_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2622
2623	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_PAE, PGM_TYPE_PROT)];
2624	pModeData->uShwType = PGM_TYPE_PAE;
2625	pModeData->uGstType = PGM_TYPE_PROT;
2626	rc = PGM_SHW_NAME_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2627	rc = PGM_GST_NAME_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2628	rc = PGM_BTH_NAME_PAE_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2629
2630	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_PAE, PGM_TYPE_32BIT)];
2631	pModeData->uShwType = PGM_TYPE_PAE;
2632	pModeData->uGstType = PGM_TYPE_32BIT;
2633	rc = PGM_SHW_NAME_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2634	rc = PGM_GST_NAME_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2635	rc = PGM_BTH_NAME_PAE_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2636
2637	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_PAE, PGM_TYPE_PAE)];
2638	pModeData->uShwType = PGM_TYPE_PAE;
2639	pModeData->uGstType = PGM_TYPE_PAE;
2640	rc = PGM_SHW_NAME_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2641	rc = PGM_GST_NAME_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2642	rc = PGM_BTH_NAME_PAE_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2643
2644	#ifdef VBOX_WITH_64_BITS_GUESTS
2645	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_AMD64, PGM_TYPE_AMD64)];
2646	pModeData->uShwType = PGM_TYPE_AMD64;
2647	pModeData->uGstType = PGM_TYPE_AMD64;
2648	rc = PGM_SHW_NAME_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2649	rc = PGM_GST_NAME_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2650	rc = PGM_BTH_NAME_AMD64_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2651	#endif
2652
2653	/* The nested paging mode. */
2654	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_REAL)];
2655	pModeData->uShwType = PGM_TYPE_NESTED;
2656	pModeData->uGstType = PGM_TYPE_REAL;
2657	rc = PGM_GST_NAME_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2658	rc = PGM_BTH_NAME_NESTED_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2659
2660	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGMMODE_PROTECTED)];
2661	pModeData->uShwType = PGM_TYPE_NESTED;
2662	pModeData->uGstType = PGM_TYPE_PROT;
2663	rc = PGM_GST_NAME_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2664	rc = PGM_BTH_NAME_NESTED_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2665
2666	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_32BIT)];
2667	pModeData->uShwType = PGM_TYPE_NESTED;
2668	pModeData->uGstType = PGM_TYPE_32BIT;
2669	rc = PGM_GST_NAME_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2670	rc = PGM_BTH_NAME_NESTED_32BIT(InitData)(pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2671
2672	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_PAE)];
2673	pModeData->uShwType = PGM_TYPE_NESTED;
2674	pModeData->uGstType = PGM_TYPE_PAE;
2675	rc = PGM_GST_NAME_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2676	rc = PGM_BTH_NAME_NESTED_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2677
2678	#ifdef VBOX_WITH_64_BITS_GUESTS
2679	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_AMD64)];
2680	pModeData->uShwType = PGM_TYPE_NESTED;
2681	pModeData->uGstType = PGM_TYPE_AMD64;
2682	rc = PGM_GST_NAME_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2683	rc = PGM_BTH_NAME_NESTED_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2684	#endif
2685
2686	/* The shadow part of the nested callback mode depends on the host paging mode (AMD-V only). */
2687	switch (pVM->pgm.s.enmHostMode)
2688	{
2689	#if HC_ARCH_BITS == 32
2690	case SUPPAGINGMODE_32_BIT:
2691	case SUPPAGINGMODE_32_BIT_GLOBAL:
2692	for (unsigned i = PGM_TYPE_REAL; i <= PGM_TYPE_PAE; i++)
2693	{
2694	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, i)];
2695	rc = PGM_SHW_NAME_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2696	}
2697	# ifdef VBOX_WITH_64_BITS_GUESTS
2698	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_AMD64)];
2699	rc = PGM_SHW_NAME_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2700	# endif
2701	break;
2702
2703	case SUPPAGINGMODE_PAE:
2704	case SUPPAGINGMODE_PAE_NX:
2705	case SUPPAGINGMODE_PAE_GLOBAL:
2706	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2707	for (unsigned i = PGM_TYPE_REAL; i <= PGM_TYPE_PAE; i++)
2708	{
2709	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, i)];
2710	rc = PGM_SHW_NAME_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2711	}
2712	# ifdef VBOX_WITH_64_BITS_GUESTS
2713	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_AMD64)];
2714	rc = PGM_SHW_NAME_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2715	# endif
2716	break;
2717	#endif /* HC_ARCH_BITS == 32 */
2718
2719	#if HC_ARCH_BITS == 64 \|\| defined(RT_OS_DARWIN)
2720	case SUPPAGINGMODE_AMD64:
2721	case SUPPAGINGMODE_AMD64_GLOBAL:
2722	case SUPPAGINGMODE_AMD64_NX:
2723	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2724	# ifdef VBOX_WITH_64_BITS_GUESTS
2725	for (unsigned i = PGM_TYPE_REAL; i <= PGM_TYPE_AMD64; i++)
2726	# else
2727	for (unsigned i = PGM_TYPE_REAL; i <= PGM_TYPE_PAE; i++)
2728	# endif
2729	{
2730	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, i)];
2731	rc = PGM_SHW_NAME_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2732	}
2733	break;
2734	#endif /* HC_ARCH_BITS == 64 \|\| RT_OS_DARWIN */
2735
2736	default:
2737	AssertFailed();
2738	break;
2739	}
2740
2741	/* Extended paging (EPT) / Intel VT-x */
2742	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_REAL)];
2743	pModeData->uShwType = PGM_TYPE_EPT;
2744	pModeData->uGstType = PGM_TYPE_REAL;
2745	rc = PGM_SHW_NAME_EPT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2746	rc = PGM_GST_NAME_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2747	rc = PGM_BTH_NAME_EPT_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2748
2749	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_PROT)];
2750	pModeData->uShwType = PGM_TYPE_EPT;
2751	pModeData->uGstType = PGM_TYPE_PROT;
2752	rc = PGM_SHW_NAME_EPT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2753	rc = PGM_GST_NAME_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2754	rc = PGM_BTH_NAME_EPT_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2755
2756	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_32BIT)];
2757	pModeData->uShwType = PGM_TYPE_EPT;
2758	pModeData->uGstType = PGM_TYPE_32BIT;
2759	rc = PGM_SHW_NAME_EPT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2760	rc = PGM_GST_NAME_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2761	rc = PGM_BTH_NAME_EPT_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2762
2763	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_PAE)];
2764	pModeData->uShwType = PGM_TYPE_EPT;
2765	pModeData->uGstType = PGM_TYPE_PAE;
2766	rc = PGM_SHW_NAME_EPT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2767	rc = PGM_GST_NAME_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2768	rc = PGM_BTH_NAME_EPT_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2769
2770	#ifdef VBOX_WITH_64_BITS_GUESTS
2771	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_AMD64)];
2772	pModeData->uShwType = PGM_TYPE_EPT;
2773	pModeData->uGstType = PGM_TYPE_AMD64;
2774	rc = PGM_SHW_NAME_EPT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2775	rc = PGM_GST_NAME_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2776	rc = PGM_BTH_NAME_EPT_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
2777	#endif
2778	return VINF_SUCCESS;
2779	}
2780
2781
2782	/**
2783	* Switch to different (or relocated in the relocate case) mode data.
2784	*
2785	* @param pVM The VM handle.
2786	* @param pVCpu The VMCPU to operate on.
2787	* @param enmShw The the shadow paging mode.
2788	* @param enmGst The the guest paging mode.
2789	*/
2790	static void pgmR3ModeDataSwitch(PVM pVM, PVMCPU pVCpu, PGMMODE enmShw, PGMMODE enmGst)
2791	{
2792	PPGMMODEDATA pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndexByMode(enmShw, enmGst)];
2793
2794	Assert(pModeData->uGstType == pgmModeToType(enmGst));
2795	Assert(pModeData->uShwType == pgmModeToType(enmShw));
2796
2797	/* shadow */
2798	pVCpu->pgm.s.pfnR3ShwRelocate = pModeData->pfnR3ShwRelocate;
2799	pVCpu->pgm.s.pfnR3ShwExit = pModeData->pfnR3ShwExit;
2800	pVCpu->pgm.s.pfnR3ShwGetPage = pModeData->pfnR3ShwGetPage;
2801	Assert(pVCpu->pgm.s.pfnR3ShwGetPage);
2802	pVCpu->pgm.s.pfnR3ShwModifyPage = pModeData->pfnR3ShwModifyPage;
2803
2804	pVCpu->pgm.s.pfnRCShwGetPage = pModeData->pfnRCShwGetPage;
2805	pVCpu->pgm.s.pfnRCShwModifyPage = pModeData->pfnRCShwModifyPage;
2806
2807	pVCpu->pgm.s.pfnR0ShwGetPage = pModeData->pfnR0ShwGetPage;
2808	pVCpu->pgm.s.pfnR0ShwModifyPage = pModeData->pfnR0ShwModifyPage;
2809
2810
2811	/* guest */
2812	pVCpu->pgm.s.pfnR3GstRelocate = pModeData->pfnR3GstRelocate;
2813	pVCpu->pgm.s.pfnR3GstExit = pModeData->pfnR3GstExit;
2814	pVCpu->pgm.s.pfnR3GstGetPage = pModeData->pfnR3GstGetPage;
2815	Assert(pVCpu->pgm.s.pfnR3GstGetPage);
2816	pVCpu->pgm.s.pfnR3GstModifyPage = pModeData->pfnR3GstModifyPage;
2817	pVCpu->pgm.s.pfnR3GstGetPDE = pModeData->pfnR3GstGetPDE;
2818	pVCpu->pgm.s.pfnRCGstGetPage = pModeData->pfnRCGstGetPage;
2819	pVCpu->pgm.s.pfnRCGstModifyPage = pModeData->pfnRCGstModifyPage;
2820	pVCpu->pgm.s.pfnRCGstGetPDE = pModeData->pfnRCGstGetPDE;
2821	pVCpu->pgm.s.pfnR0GstGetPage = pModeData->pfnR0GstGetPage;
2822	pVCpu->pgm.s.pfnR0GstModifyPage = pModeData->pfnR0GstModifyPage;
2823	pVCpu->pgm.s.pfnR0GstGetPDE = pModeData->pfnR0GstGetPDE;
2824
2825	/* both */
2826	pVCpu->pgm.s.pfnR3BthRelocate = pModeData->pfnR3BthRelocate;
2827	pVCpu->pgm.s.pfnR3BthInvalidatePage = pModeData->pfnR3BthInvalidatePage;
2828	pVCpu->pgm.s.pfnR3BthSyncCR3 = pModeData->pfnR3BthSyncCR3;
2829	Assert(pVCpu->pgm.s.pfnR3BthSyncCR3);
2830	pVCpu->pgm.s.pfnR3BthSyncPage = pModeData->pfnR3BthSyncPage;
2831	pVCpu->pgm.s.pfnR3BthPrefetchPage = pModeData->pfnR3BthPrefetchPage;
2832	pVCpu->pgm.s.pfnR3BthVerifyAccessSyncPage = pModeData->pfnR3BthVerifyAccessSyncPage;
2833	#ifdef VBOX_STRICT
2834	pVCpu->pgm.s.pfnR3BthAssertCR3 = pModeData->pfnR3BthAssertCR3;
2835	#endif
2836	pVCpu->pgm.s.pfnR3BthMapCR3 = pModeData->pfnR3BthMapCR3;
2837	pVCpu->pgm.s.pfnR3BthUnmapCR3 = pModeData->pfnR3BthUnmapCR3;
2838
2839	pVCpu->pgm.s.pfnRCBthTrap0eHandler = pModeData->pfnRCBthTrap0eHandler;
2840	pVCpu->pgm.s.pfnRCBthInvalidatePage = pModeData->pfnRCBthInvalidatePage;
2841	pVCpu->pgm.s.pfnRCBthSyncCR3 = pModeData->pfnRCBthSyncCR3;
2842	pVCpu->pgm.s.pfnRCBthSyncPage = pModeData->pfnRCBthSyncPage;
2843	pVCpu->pgm.s.pfnRCBthPrefetchPage = pModeData->pfnRCBthPrefetchPage;
2844	pVCpu->pgm.s.pfnRCBthVerifyAccessSyncPage = pModeData->pfnRCBthVerifyAccessSyncPage;
2845	#ifdef VBOX_STRICT
2846	pVCpu->pgm.s.pfnRCBthAssertCR3 = pModeData->pfnRCBthAssertCR3;
2847	#endif
2848	pVCpu->pgm.s.pfnRCBthMapCR3 = pModeData->pfnRCBthMapCR3;
2849	pVCpu->pgm.s.pfnRCBthUnmapCR3 = pModeData->pfnRCBthUnmapCR3;
2850
2851	pVCpu->pgm.s.pfnR0BthTrap0eHandler = pModeData->pfnR0BthTrap0eHandler;
2852	pVCpu->pgm.s.pfnR0BthInvalidatePage = pModeData->pfnR0BthInvalidatePage;
2853	pVCpu->pgm.s.pfnR0BthSyncCR3 = pModeData->pfnR0BthSyncCR3;
2854	pVCpu->pgm.s.pfnR0BthSyncPage = pModeData->pfnR0BthSyncPage;
2855	pVCpu->pgm.s.pfnR0BthPrefetchPage = pModeData->pfnR0BthPrefetchPage;
2856	pVCpu->pgm.s.pfnR0BthVerifyAccessSyncPage = pModeData->pfnR0BthVerifyAccessSyncPage;
2857	#ifdef VBOX_STRICT
2858	pVCpu->pgm.s.pfnR0BthAssertCR3 = pModeData->pfnR0BthAssertCR3;
2859	#endif
2860	pVCpu->pgm.s.pfnR0BthMapCR3 = pModeData->pfnR0BthMapCR3;
2861	pVCpu->pgm.s.pfnR0BthUnmapCR3 = pModeData->pfnR0BthUnmapCR3;
2862	}
2863
2864
2865	/**
2866	* Calculates the shadow paging mode.
2867	*
2868	* @returns The shadow paging mode.
2869	* @param pVM VM handle.
2870	* @param enmGuestMode The guest mode.
2871	* @param enmHostMode The host mode.
2872	* @param enmShadowMode The current shadow mode.
2873	* @param penmSwitcher Where to store the switcher to use.
2874	* VMMSWITCHER_INVALID means no change.
2875	*/
2876	static PGMMODE pgmR3CalcShadowMode(PVM pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode, VMMSWITCHER *penmSwitcher)
2877	{
2878	VMMSWITCHER enmSwitcher = VMMSWITCHER_INVALID;
2879	switch (enmGuestMode)
2880	{
2881	/*
2882	* When switching to real or protected mode we don't change
2883	* anything since it's likely that we'll switch back pretty soon.
2884	*
2885	* During pgmR3InitPaging we'll end up here with PGMMODE_INVALID
2886	* and is supposed to determine which shadow paging and switcher to
2887	* use during init.
2888	*/
2889	case PGMMODE_REAL:
2890	case PGMMODE_PROTECTED:
2891	if ( enmShadowMode != PGMMODE_INVALID
2892	&& !HWACCMIsEnabled(pVM) /* always switch in hwaccm mode! */)
2893	break; /* (no change) */
2894
2895	switch (enmHostMode)
2896	{
2897	case SUPPAGINGMODE_32_BIT:
2898	case SUPPAGINGMODE_32_BIT_GLOBAL:
2899	enmShadowMode = PGMMODE_32_BIT;
2900	enmSwitcher = VMMSWITCHER_32_TO_32;
2901	break;
2902
2903	case SUPPAGINGMODE_PAE:
2904	case SUPPAGINGMODE_PAE_NX:
2905	case SUPPAGINGMODE_PAE_GLOBAL:
2906	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2907	enmShadowMode = PGMMODE_PAE;
2908	enmSwitcher = VMMSWITCHER_PAE_TO_PAE;
2909	#ifdef DEBUG_bird
2910	if (RTEnvExist("VBOX_32BIT"))
2911	{
2912	enmShadowMode = PGMMODE_32_BIT;
2913	enmSwitcher = VMMSWITCHER_PAE_TO_32;
2914	}
2915	#endif
2916	break;
2917
2918	case SUPPAGINGMODE_AMD64:
2919	case SUPPAGINGMODE_AMD64_GLOBAL:
2920	case SUPPAGINGMODE_AMD64_NX:
2921	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2922	enmShadowMode = PGMMODE_PAE;
2923	enmSwitcher = VMMSWITCHER_AMD64_TO_PAE;
2924	#ifdef DEBUG_bird
2925	if (RTEnvExist("VBOX_32BIT"))
2926	{
2927	enmShadowMode = PGMMODE_32_BIT;
2928	enmSwitcher = VMMSWITCHER_AMD64_TO_32;
2929	}
2930	#endif
2931	break;
2932
2933	default: AssertMsgFailed(("enmHostMode=%d\n", enmHostMode)); break;
2934	}
2935	break;
2936
2937	case PGMMODE_32_BIT:
2938	switch (enmHostMode)
2939	{
2940	case SUPPAGINGMODE_32_BIT:
2941	case SUPPAGINGMODE_32_BIT_GLOBAL:
2942	enmShadowMode = PGMMODE_32_BIT;
2943	enmSwitcher = VMMSWITCHER_32_TO_32;
2944	break;
2945
2946	case SUPPAGINGMODE_PAE:
2947	case SUPPAGINGMODE_PAE_NX:
2948	case SUPPAGINGMODE_PAE_GLOBAL:
2949	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2950	enmShadowMode = PGMMODE_PAE;
2951	enmSwitcher = VMMSWITCHER_PAE_TO_PAE;
2952	#ifdef DEBUG_bird
2953	if (RTEnvExist("VBOX_32BIT"))
2954	{
2955	enmShadowMode = PGMMODE_32_BIT;
2956	enmSwitcher = VMMSWITCHER_PAE_TO_32;
2957	}
2958	#endif
2959	break;
2960
2961	case SUPPAGINGMODE_AMD64:
2962	case SUPPAGINGMODE_AMD64_GLOBAL:
2963	case SUPPAGINGMODE_AMD64_NX:
2964	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2965	enmShadowMode = PGMMODE_PAE;
2966	enmSwitcher = VMMSWITCHER_AMD64_TO_PAE;
2967	#ifdef DEBUG_bird
2968	if (RTEnvExist("VBOX_32BIT"))
2969	{
2970	enmShadowMode = PGMMODE_32_BIT;
2971	enmSwitcher = VMMSWITCHER_AMD64_TO_32;
2972	}
2973	#endif
2974	break;
2975
2976	default: AssertMsgFailed(("enmHostMode=%d\n", enmHostMode)); break;
2977	}
2978	break;
2979
2980	case PGMMODE_PAE:
2981	case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
2982	switch (enmHostMode)
2983	{
2984	case SUPPAGINGMODE_32_BIT:
2985	case SUPPAGINGMODE_32_BIT_GLOBAL:
2986	enmShadowMode = PGMMODE_PAE;
2987	enmSwitcher = VMMSWITCHER_32_TO_PAE;
2988	break;
2989
2990	case SUPPAGINGMODE_PAE:
2991	case SUPPAGINGMODE_PAE_NX:
2992	case SUPPAGINGMODE_PAE_GLOBAL:
2993	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2994	enmShadowMode = PGMMODE_PAE;
2995	enmSwitcher = VMMSWITCHER_PAE_TO_PAE;
2996	break;
2997
2998	case SUPPAGINGMODE_AMD64:
2999	case SUPPAGINGMODE_AMD64_GLOBAL:
3000	case SUPPAGINGMODE_AMD64_NX:
3001	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3002	enmShadowMode = PGMMODE_PAE;
3003	enmSwitcher = VMMSWITCHER_AMD64_TO_PAE;
3004	break;
3005
3006	default: AssertMsgFailed(("enmHostMode=%d\n", enmHostMode)); break;
3007	}
3008	break;
3009
3010	case PGMMODE_AMD64:
3011	case PGMMODE_AMD64_NX:
3012	switch (enmHostMode)
3013	{
3014	case SUPPAGINGMODE_32_BIT:
3015	case SUPPAGINGMODE_32_BIT_GLOBAL:
3016	enmShadowMode = PGMMODE_AMD64;
3017	enmSwitcher = VMMSWITCHER_32_TO_AMD64;
3018	break;
3019
3020	case SUPPAGINGMODE_PAE:
3021	case SUPPAGINGMODE_PAE_NX:
3022	case SUPPAGINGMODE_PAE_GLOBAL:
3023	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3024	enmShadowMode = PGMMODE_AMD64;
3025	enmSwitcher = VMMSWITCHER_PAE_TO_AMD64;
3026	break;
3027
3028	case SUPPAGINGMODE_AMD64:
3029	case SUPPAGINGMODE_AMD64_GLOBAL:
3030	case SUPPAGINGMODE_AMD64_NX:
3031	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3032	enmShadowMode = PGMMODE_AMD64;
3033	enmSwitcher = VMMSWITCHER_AMD64_TO_AMD64;
3034	break;
3035
3036	default: AssertMsgFailed(("enmHostMode=%d\n", enmHostMode)); break;
3037	}
3038	break;
3039
3040
3041	default:
3042	AssertReleaseMsgFailed(("enmGuestMode=%d\n", enmGuestMode));
3043	*penmSwitcher = VMMSWITCHER_INVALID;
3044	return PGMMODE_INVALID;
3045	}
3046	/* Override the shadow mode is nested paging is active. */
3047	if (HWACCMIsNestedPagingActive(pVM))
3048	enmShadowMode = HWACCMGetShwPagingMode(pVM);
3049
3050	*penmSwitcher = enmSwitcher;
3051	return enmShadowMode;
3052	}
3053
3054
3055	/**
3056	* Performs the actual mode change.
3057	* This is called by PGMChangeMode and pgmR3InitPaging().
3058	*
3059	* @returns VBox status code. May suspend or power off the VM on error, but this
3060	* will trigger using FFs and not status codes.
3061	*
3062	* @param pVM VM handle.
3063	* @param pVCpu The VMCPU to operate on.
3064	* @param enmGuestMode The new guest mode. This is assumed to be different from
3065	* the current mode.
3066	*/
3067	VMMR3DECL(int) PGMR3ChangeMode(PVM pVM, PVMCPU pVCpu, PGMMODE enmGuestMode)
3068	{
3069	bool fIsOldGuestPagingMode64Bits = (pVCpu->pgm.s.enmGuestMode >= PGMMODE_AMD64);
3070	bool fIsNewGuestPagingMode64Bits = (enmGuestMode >= PGMMODE_AMD64);
3071
3072	Log(("PGMR3ChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
3073	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
3074
3075	/*
3076	* Calc the shadow mode and switcher.
3077	*/
3078	VMMSWITCHER enmSwitcher;
3079	PGMMODE enmShadowMode = pgmR3CalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode, &enmSwitcher);
3080
3081	#ifdef VBOX_WITH_RAW_MODE
3082	if (enmSwitcher != VMMSWITCHER_INVALID)
3083	{
3084	/*
3085	* Select new switcher.
3086	*/
3087	int rc = VMMR3SelectSwitcher(pVM, enmSwitcher);
3088	if (RT_FAILURE(rc))
3089	{
3090	AssertReleaseMsgFailed(("VMMR3SelectSwitcher(%d) -> %Rrc\n", enmSwitcher, rc));
3091	return rc;
3092	}
3093	}
3094	#endif
3095
3096	/*
3097	* Exit old mode(s).
3098	*/
3099	#if HC_ARCH_BITS == 32
3100	/* The nested shadow paging mode for AMD-V does change when running 64 bits guests on 32 bits hosts; typically PAE <-> AMD64 */
3101	const bool fForceShwEnterExit = ( fIsOldGuestPagingMode64Bits != fIsNewGuestPagingMode64Bits
3102	&& enmShadowMode == PGMMODE_NESTED);
3103	#else
3104	const bool fForceShwEnterExit = false;
3105	#endif
3106	/* shadow */
3107	if ( enmShadowMode != pVCpu->pgm.s.enmShadowMode
3108	\|\| fForceShwEnterExit)
3109	{
3110	LogFlow(("PGMR3ChangeMode: Shadow mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
3111	if (PGM_SHW_PFN(Exit, pVCpu))
3112	{
3113	int rc = PGM_SHW_PFN(Exit, pVCpu)(pVCpu);
3114	if (RT_FAILURE(rc))
3115	{
3116	AssertMsgFailed(("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc));
3117	return rc;
3118	}
3119	}
3120
3121	}
3122	else
3123	LogFlow(("PGMR3ChangeMode: Shadow mode remains: %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
3124
3125	/* guest */
3126	if (PGM_GST_PFN(Exit, pVCpu))
3127	{
3128	int rc = PGM_GST_PFN(Exit, pVCpu)(pVCpu);
3129	if (RT_FAILURE(rc))
3130	{
3131	AssertMsgFailed(("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc));
3132	return rc;
3133	}
3134	}
3135
3136	/*
3137	* Load new paging mode data.
3138	*/
3139	pgmR3ModeDataSwitch(pVM, pVCpu, enmShadowMode, enmGuestMode);
3140
3141	/*
3142	* Enter new shadow mode (if changed).
3143	*/
3144	if ( enmShadowMode != pVCpu->pgm.s.enmShadowMode
3145	\|\| fForceShwEnterExit)
3146	{
3147	int rc;
3148	pVCpu->pgm.s.enmShadowMode = enmShadowMode;
3149	switch (enmShadowMode)
3150	{
3151	case PGMMODE_32_BIT:
3152	rc = PGM_SHW_NAME_32BIT(Enter)(pVCpu, false);
3153	break;
3154	case PGMMODE_PAE:
3155	case PGMMODE_PAE_NX:
3156	rc = PGM_SHW_NAME_PAE(Enter)(pVCpu, false);
3157	break;
3158	case PGMMODE_AMD64:
3159	case PGMMODE_AMD64_NX:
3160	rc = PGM_SHW_NAME_AMD64(Enter)(pVCpu, fIsNewGuestPagingMode64Bits);
3161	break;
3162	case PGMMODE_NESTED:
3163	rc = PGM_SHW_NAME_NESTED(Enter)(pVCpu, fIsNewGuestPagingMode64Bits);
3164	break;
3165	case PGMMODE_EPT:
3166	rc = PGM_SHW_NAME_EPT(Enter)(pVCpu, fIsNewGuestPagingMode64Bits);
3167	break;
3168	case PGMMODE_REAL:
3169	case PGMMODE_PROTECTED:
3170	default:
3171	AssertReleaseMsgFailed(("enmShadowMode=%d\n", enmShadowMode));
3172	return VERR_INTERNAL_ERROR;
3173	}
3174	if (RT_FAILURE(rc))
3175	{
3176	AssertReleaseMsgFailed(("Entering enmShadowMode=%d failed: %Rrc\n", enmShadowMode, rc));
3177	pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID;
3178	return rc;
3179	}
3180	}
3181
3182	/*
3183	* Always flag the necessary updates
3184	*/
3185	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3186
3187	/*
3188	* Enter the new guest and shadow+guest modes.
3189	*/
3190	int rc = -1;
3191	int rc2 = -1;
3192	RTGCPHYS GCPhysCR3 = NIL_RTGCPHYS;
3193	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3194	switch (enmGuestMode)
3195	{
3196	case PGMMODE_REAL:
3197	rc = PGM_GST_NAME_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
3198	switch (pVCpu->pgm.s.enmShadowMode)
3199	{
3200	case PGMMODE_32_BIT:
3201	rc2 = PGM_BTH_NAME_32BIT_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
3202	break;
3203	case PGMMODE_PAE:
3204	case PGMMODE_PAE_NX:
3205	rc2 = PGM_BTH_NAME_PAE_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
3206	break;
3207	case PGMMODE_NESTED:
3208	rc2 = PGM_BTH_NAME_NESTED_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
3209	break;
3210	case PGMMODE_EPT:
3211	rc2 = PGM_BTH_NAME_EPT_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
3212	break;
3213	case PGMMODE_AMD64:
3214	case PGMMODE_AMD64_NX:
3215	AssertMsgFailed(("Should use PAE shadow mode!\n"));
3216	default: AssertFailed(); break;
3217	}
3218	break;
3219
3220	case PGMMODE_PROTECTED:
3221	rc = PGM_GST_NAME_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
3222	switch (pVCpu->pgm.s.enmShadowMode)
3223	{
3224	case PGMMODE_32_BIT:
3225	rc2 = PGM_BTH_NAME_32BIT_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
3226	break;
3227	case PGMMODE_PAE:
3228	case PGMMODE_PAE_NX:
3229	rc2 = PGM_BTH_NAME_PAE_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
3230	break;
3231	case PGMMODE_NESTED:
3232	rc2 = PGM_BTH_NAME_NESTED_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
3233	break;
3234	case PGMMODE_EPT:
3235	rc2 = PGM_BTH_NAME_EPT_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
3236	break;
3237	case PGMMODE_AMD64:
3238	case PGMMODE_AMD64_NX:
3239	AssertMsgFailed(("Should use PAE shadow mode!\n"));
3240	default: AssertFailed(); break;
3241	}
3242	break;
3243
3244	case PGMMODE_32_BIT:
3245	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
3246	rc = PGM_GST_NAME_32BIT(Enter)(pVCpu, GCPhysCR3);
3247	switch (pVCpu->pgm.s.enmShadowMode)
3248	{
3249	case PGMMODE_32_BIT:
3250	rc2 = PGM_BTH_NAME_32BIT_32BIT(Enter)(pVCpu, GCPhysCR3);
3251	break;
3252	case PGMMODE_PAE:
3253	case PGMMODE_PAE_NX:
3254	rc2 = PGM_BTH_NAME_PAE_32BIT(Enter)(pVCpu, GCPhysCR3);
3255	break;
3256	case PGMMODE_NESTED:
3257	rc2 = PGM_BTH_NAME_NESTED_32BIT(Enter)(pVCpu, GCPhysCR3);
3258	break;
3259	case PGMMODE_EPT:
3260	rc2 = PGM_BTH_NAME_EPT_32BIT(Enter)(pVCpu, GCPhysCR3);
3261	break;
3262	case PGMMODE_AMD64:
3263	case PGMMODE_AMD64_NX:
3264	AssertMsgFailed(("Should use PAE shadow mode!\n"));
3265	default: AssertFailed(); break;
3266	}
3267	break;
3268
3269	case PGMMODE_PAE_NX:
3270	case PGMMODE_PAE:
3271	{
3272	uint32_t u32Dummy, u32Features;
3273
3274	CPUMGetGuestCpuId(pVCpu, 1, &u32Dummy, &u32Dummy, &u32Dummy, &u32Features);
3275	if (!(u32Features & X86_CPUID_FEATURE_EDX_PAE))
3276	return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
3277	N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (General/Advanced)"));
3278
3279	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
3280	rc = PGM_GST_NAME_PAE(Enter)(pVCpu, GCPhysCR3);
3281	switch (pVCpu->pgm.s.enmShadowMode)
3282	{
3283	case PGMMODE_PAE:
3284	case PGMMODE_PAE_NX:
3285	rc2 = PGM_BTH_NAME_PAE_PAE(Enter)(pVCpu, GCPhysCR3);
3286	break;
3287	case PGMMODE_NESTED:
3288	rc2 = PGM_BTH_NAME_NESTED_PAE(Enter)(pVCpu, GCPhysCR3);
3289	break;
3290	case PGMMODE_EPT:
3291	rc2 = PGM_BTH_NAME_EPT_PAE(Enter)(pVCpu, GCPhysCR3);
3292	break;
3293	case PGMMODE_32_BIT:
3294	case PGMMODE_AMD64:
3295	case PGMMODE_AMD64_NX:
3296	AssertMsgFailed(("Should use PAE shadow mode!\n"));
3297	default: AssertFailed(); break;
3298	}
3299	break;
3300	}
3301
3302	#ifdef VBOX_WITH_64_BITS_GUESTS
3303	case PGMMODE_AMD64_NX:
3304	case PGMMODE_AMD64:
3305	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & UINT64_C(0xfffffffffffff000); /** @todo define this mask! */
3306	rc = PGM_GST_NAME_AMD64(Enter)(pVCpu, GCPhysCR3);
3307	switch (pVCpu->pgm.s.enmShadowMode)
3308	{
3309	case PGMMODE_AMD64:
3310	case PGMMODE_AMD64_NX:
3311	rc2 = PGM_BTH_NAME_AMD64_AMD64(Enter)(pVCpu, GCPhysCR3);
3312	break;
3313	case PGMMODE_NESTED:
3314	rc2 = PGM_BTH_NAME_NESTED_AMD64(Enter)(pVCpu, GCPhysCR3);
3315	break;
3316	case PGMMODE_EPT:
3317	rc2 = PGM_BTH_NAME_EPT_AMD64(Enter)(pVCpu, GCPhysCR3);
3318	break;
3319	case PGMMODE_32_BIT:
3320	case PGMMODE_PAE:
3321	case PGMMODE_PAE_NX:
3322	AssertMsgFailed(("Should use AMD64 shadow mode!\n"));
3323	default: AssertFailed(); break;
3324	}
3325	break;
3326	#endif
3327
3328	default:
3329	AssertReleaseMsgFailed(("enmGuestMode=%d\n", enmGuestMode));
3330	rc = VERR_NOT_IMPLEMENTED;
3331	break;
3332	}
3333
3334	/* status codes. */
3335	AssertRC(rc);
3336	AssertRC(rc2);
3337	if (RT_SUCCESS(rc))
3338	{
3339	rc = rc2;
3340	if (RT_SUCCESS(rc)) /* no informational status codes. */
3341	rc = VINF_SUCCESS;
3342	}
3343
3344	/* Notify HWACCM as well. */
3345	HWACCMR3PagingModeChanged(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3346	return rc;
3347	}
3348
3349	/**
3350	* Release the pgm lock if owned by the current VCPU
3351	*
3352	* @param pVM The VM to operate on.
3353	*/
3354	VMMR3DECL(void) PGMR3ReleaseOwnedLocks(PVM pVM)
3355	{
3356	while (PDMCritSectIsOwner(&pVM->pgm.s.CritSect))
3357	PDMCritSectLeave(&pVM->pgm.s.CritSect);
3358	}
3359
3360	/**
3361	* Called by pgmPoolFlushAllInt prior to flushing the pool.
3362	*
3363	* @returns VBox status code, fully asserted.
3364	* @param pVM The VM handle.
3365	* @param pVCpu The VMCPU to operate on.
3366	*/
3367	int pgmR3ExitShadowModeBeforePoolFlush(PVM pVM, PVMCPU pVCpu)
3368	{
3369	/* Unmap the old CR3 value before flushing everything. */
3370	int rc = PGM_BTH_PFN(UnmapCR3, pVCpu)(pVCpu);
3371	AssertRC(rc);
3372
3373	/* Exit the current shadow paging mode as well; nested paging and EPT use a root CR3 which will get flushed here. */
3374	rc = PGM_SHW_PFN(Exit, pVCpu)(pVCpu);
3375	AssertRC(rc);
3376	Assert(pVCpu->pgm.s.pShwPageCR3R3 == NULL);
3377	return rc;
3378	}
3379
3380
3381	/**
3382	* Called by pgmPoolFlushAllInt after flushing the pool.
3383	*
3384	* @returns VBox status code, fully asserted.
3385	* @param pVM The VM handle.
3386	* @param pVCpu The VMCPU to operate on.
3387	*/
3388	int pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu)
3389	{
3390	pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID;
3391	int rc = PGMR3ChangeMode(pVM, pVCpu, PGMGetGuestMode(pVCpu));
3392	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3393	AssertRCReturn(rc, rc);
3394	AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
3395
3396	Assert(pVCpu->pgm.s.pShwPageCR3R3 != NULL);
3397	AssertMsg( pVCpu->pgm.s.enmShadowMode >= PGMMODE_NESTED
3398	\|\| CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu),
3399	("%RHp != %RHp %s\n", (RTHCPHYS)CPUMGetHyperCR3(pVCpu), PGMGetHyperCR3(pVCpu), PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
3400	return rc;
3401	}
3402
3403
3404	/**
3405	* Dumps a PAE shadow page table.
3406	*
3407	* @returns VBox status code (VINF_SUCCESS).
3408	* @param pVM The VM handle.
3409	* @param pPT Pointer to the page table.
3410	* @param u64Address The virtual address of the page table starts.
3411	* @param fLongMode Set if this a long mode table; clear if it's a legacy mode table.
3412	* @param cMaxDepth The maxium depth.
3413	* @param pHlp Pointer to the output functions.
3414	*/
3415	static int pgmR3DumpHierarchyHCPaePT(PVM pVM, PX86PTPAE pPT, uint64_t u64Address, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp)
3416	{
3417	for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
3418	{
3419	X86PTEPAE Pte = pPT->a[i];
3420	if (Pte.n.u1Present)
3421	{
3422	pHlp->pfnPrintf(pHlp,
3423	fLongMode /P R S A D G WT CD AT NX 4M a p ? /
3424	? "%016llx 3 \| P %c %c %c %c %c %s %s %s %s 4K %c%c%c %016llx\n"
3425	: "%08llx 2 \| P %c %c %c %c %c %s %s %s %s 4K %c%c%c %016llx\n",
3426	u64Address + ((uint64_t)i << X86_PT_PAE_SHIFT),
3427	Pte.n.u1Write ? 'W' : 'R',
3428	Pte.n.u1User ? 'U' : 'S',
3429	Pte.n.u1Accessed ? 'A' : '-',
3430	Pte.n.u1Dirty ? 'D' : '-',
3431	Pte.n.u1Global ? 'G' : '-',
3432	Pte.n.u1WriteThru ? "WT" : "--",
3433	Pte.n.u1CacheDisable? "CD" : "--",
3434	Pte.n.u1PAT ? "AT" : "--",
3435	Pte.n.u1NoExecute ? "NX" : "--",
3436	Pte.u & PGM_PTFLAGS_TRACK_DIRTY ? 'd' : '-',
3437	Pte.u & RT_BIT(10) ? '1' : '0',
3438	Pte.u & PGM_PTFLAGS_CSAM_VALIDATED? 'v' : '-',
3439	Pte.u & X86_PTE_PAE_PG_MASK);
3440	}
3441	}
3442	return VINF_SUCCESS;
3443	}
3444
3445
3446	/**
3447	* Dumps a PAE shadow page directory table.
3448	*
3449	* @returns VBox status code (VINF_SUCCESS).
3450	* @param pVM The VM handle.
3451	* @param HCPhys The physical address of the page directory table.
3452	* @param u64Address The virtual address of the page table starts.
3453	* @param cr4 The CR4, PSE is currently used.
3454	* @param fLongMode Set if this a long mode table; clear if it's a legacy mode table.
3455	* @param cMaxDepth The maxium depth.
3456	* @param pHlp Pointer to the output functions.
3457	*/
3458	static int pgmR3DumpHierarchyHCPaePD(PVM pVM, RTHCPHYS HCPhys, uint64_t u64Address, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp)
3459	{
3460	PX86PDPAE pPD = (PX86PDPAE)MMPagePhys2Page(pVM, HCPhys);
3461	if (!pPD)
3462	{
3463	pHlp->pfnPrintf(pHlp, "%0*llx error! Page directory at HCPhys=%RHp was not found in the page pool!\n",
3464	fLongMode ? 16 : 8, u64Address, HCPhys);
3465	return VERR_INVALID_PARAMETER;
3466	}
3467	const bool fBigPagesSupported = fLongMode \|\| !!(cr4 & X86_CR4_PSE);
3468
3469	int rc = VINF_SUCCESS;
3470	for (unsigned i = 0; i < RT_ELEMENTS(pPD->a); i++)
3471	{
3472	X86PDEPAE Pde = pPD->a[i];
3473	if (Pde.n.u1Present)
3474	{
3475	if (fBigPagesSupported && Pde.b.u1Size)
3476	pHlp->pfnPrintf(pHlp,
3477	fLongMode /P R S A D G WT CD AT NX 4M a p ? /
3478	? "%016llx 2 \| P %c %c %c %c %c %s %s %s %s 4M %c%c%c %016llx\n"
3479	: "%08llx 1 \| P %c %c %c %c %c %s %s %s %s 4M %c%c%c %016llx\n",
3480	u64Address + ((uint64_t)i << X86_PD_PAE_SHIFT),
3481	Pde.b.u1Write ? 'W' : 'R',
3482	Pde.b.u1User ? 'U' : 'S',
3483	Pde.b.u1Accessed ? 'A' : '-',
3484	Pde.b.u1Dirty ? 'D' : '-',
3485	Pde.b.u1Global ? 'G' : '-',
3486	Pde.b.u1WriteThru ? "WT" : "--",
3487	Pde.b.u1CacheDisable? "CD" : "--",
3488	Pde.b.u1PAT ? "AT" : "--",
3489	Pde.b.u1NoExecute ? "NX" : "--",
3490	Pde.u & RT_BIT_64(9) ? '1' : '0',
3491	Pde.u & PGM_PDFLAGS_MAPPING ? 'm' : '-',
3492	Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
3493	Pde.u & X86_PDE_PAE_PG_MASK);
3494	else
3495	{
3496	pHlp->pfnPrintf(pHlp,
3497	fLongMode /P R S A D G WT CD AT NX 4M a p ? /
3498	? "%016llx 2 \| P %c %c %c %c %c %s %s .. %s 4K %c%c%c %016llx\n"
3499	: "%08llx 1 \| P %c %c %c %c %c %s %s .. %s 4K %c%c%c %016llx\n",
3500	u64Address + ((uint64_t)i << X86_PD_PAE_SHIFT),
3501	Pde.n.u1Write ? 'W' : 'R',
3502	Pde.n.u1User ? 'U' : 'S',
3503	Pde.n.u1Accessed ? 'A' : '-',
3504	Pde.n.u1Reserved0 ? '?' : '.', /* ignored */
3505	Pde.n.u1Reserved1 ? '?' : '.', /* ignored */
3506	Pde.n.u1WriteThru ? "WT" : "--",
3507	Pde.n.u1CacheDisable? "CD" : "--",
3508	Pde.n.u1NoExecute ? "NX" : "--",
3509	Pde.u & RT_BIT_64(9) ? '1' : '0',
3510	Pde.u & PGM_PDFLAGS_MAPPING ? 'm' : '-',
3511	Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
3512	Pde.u & X86_PDE_PAE_PG_MASK);
3513	if (cMaxDepth >= 1)
3514	{
3515	/** @todo what about using the page pool for mapping PTs? */
3516	uint64_t u64AddressPT = u64Address + ((uint64_t)i << X86_PD_PAE_SHIFT);
3517	RTHCPHYS HCPhysPT = Pde.u & X86_PDE_PAE_PG_MASK;
3518	PX86PTPAE pPT = NULL;
3519	if (!(Pde.u & PGM_PDFLAGS_MAPPING))
3520	pPT = (PX86PTPAE)MMPagePhys2Page(pVM, HCPhysPT);
3521	else
3522	{
3523	for (PPGMMAPPING pMap = pVM->pgm.s.pMappingsR3; pMap; pMap = pMap->pNextR3)
3524	{
3525	uint64_t off = u64AddressPT - pMap->GCPtr;
3526	if (off < pMap->cb)
3527	{
3528	const int iPDE = (uint32_t)(off >> X86_PD_SHIFT);
3529	const int iSub = (int)((off >> X86_PD_PAE_SHIFT) & 1); /* MSC is a pain sometimes */
3530	if ((iSub ? pMap->aPTs[iPDE].HCPhysPaePT1 : pMap->aPTs[iPDE].HCPhysPaePT0) != HCPhysPT)
3531	pHlp->pfnPrintf(pHlp, "%0*llx error! Mapping error! PT %d has HCPhysPT=%RHp not %RHp is in the PD.\n",
3532	fLongMode ? 16 : 8, u64AddressPT, iPDE,
3533	iSub ? pMap->aPTs[iPDE].HCPhysPaePT1 : pMap->aPTs[iPDE].HCPhysPaePT0, HCPhysPT);
3534	pPT = &pMap->aPTs[iPDE].paPaePTsR3[iSub];
3535	}
3536	}
3537	}
3538	int rc2 = VERR_INVALID_PARAMETER;
3539	if (pPT)
3540	rc2 = pgmR3DumpHierarchyHCPaePT(pVM, pPT, u64AddressPT, fLongMode, cMaxDepth - 1, pHlp);
3541	else
3542	pHlp->pfnPrintf(pHlp, "%0*llx error! Page table at HCPhys=%RHp was not found in the page pool!\n",
3543	fLongMode ? 16 : 8, u64AddressPT, HCPhysPT);
3544	if (rc2 < rc && RT_SUCCESS(rc))
3545	rc = rc2;
3546	}
3547	}
3548	}
3549	}
3550	return rc;
3551	}
3552
3553
3554	/**
3555	* Dumps a PAE shadow page directory pointer table.
3556	*
3557	* @returns VBox status code (VINF_SUCCESS).
3558	* @param pVM The VM handle.
3559	* @param HCPhys The physical address of the page directory pointer table.
3560	* @param u64Address The virtual address of the page table starts.
3561	* @param cr4 The CR4, PSE is currently used.
3562	* @param fLongMode Set if this a long mode table; clear if it's a legacy mode table.
3563	* @param cMaxDepth The maxium depth.
3564	* @param pHlp Pointer to the output functions.
3565	*/
3566	static int pgmR3DumpHierarchyHCPaePDPT(PVM pVM, RTHCPHYS HCPhys, uint64_t u64Address, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp)
3567	{
3568	PX86PDPT pPDPT = (PX86PDPT)MMPagePhys2Page(pVM, HCPhys);
3569	if (!pPDPT)
3570	{
3571	pHlp->pfnPrintf(pHlp, "%0*llx error! Page directory pointer table at HCPhys=%RHp was not found in the page pool!\n",
3572	fLongMode ? 16 : 8, u64Address, HCPhys);
3573	return VERR_INVALID_PARAMETER;
3574	}
3575
3576	int rc = VINF_SUCCESS;
3577	const unsigned c = fLongMode ? RT_ELEMENTS(pPDPT->a) : X86_PG_PAE_PDPE_ENTRIES;
3578	for (unsigned i = 0; i < c; i++)
3579	{
3580	X86PDPE Pdpe = pPDPT->a[i];
3581	if (Pdpe.n.u1Present)
3582	{
3583	if (fLongMode)
3584	pHlp->pfnPrintf(pHlp, /P R S A D G WT CD AT NX 4M a p ? /
3585	"%016llx 1 \| P %c %c %c %c %c %s %s %s %s .. %c%c%c %016llx\n",
3586	u64Address + ((uint64_t)i << X86_PDPT_SHIFT),
3587	Pdpe.lm.u1Write ? 'W' : 'R',
3588	Pdpe.lm.u1User ? 'U' : 'S',
3589	Pdpe.lm.u1Accessed ? 'A' : '-',
3590	Pdpe.lm.u3Reserved & 1? '?' : '.', /* ignored */
3591	Pdpe.lm.u3Reserved & 4? '!' : '.', /* mbz */
3592	Pdpe.lm.u1WriteThru ? "WT" : "--",
3593	Pdpe.lm.u1CacheDisable? "CD" : "--",
3594	Pdpe.lm.u3Reserved & 2? "!" : "..",/* mbz */
3595	Pdpe.lm.u1NoExecute ? "NX" : "--",
3596	Pdpe.u & RT_BIT(9) ? '1' : '0',
3597	Pdpe.u & PGM_PLXFLAGS_PERMANENT ? 'p' : '-',
3598	Pdpe.u & RT_BIT(11) ? '1' : '0',
3599	Pdpe.u & X86_PDPE_PG_MASK);
3600	else
3601	pHlp->pfnPrintf(pHlp, /P G WT CD AT NX 4M a p ? /
3602	"%08x 0 \| P %c %s %s %s %s .. %c%c%c %016llx\n",
3603	i << X86_PDPT_SHIFT,
3604	Pdpe.n.u4Reserved & 1? '!' : '.', /* mbz */
3605	Pdpe.n.u4Reserved & 4? '!' : '.', /* mbz */
3606	Pdpe.n.u1WriteThru ? "WT" : "--",
3607	Pdpe.n.u1CacheDisable? "CD" : "--",
3608	Pdpe.n.u4Reserved & 2? "!" : "..",/* mbz */
3609	Pdpe.u & RT_BIT(9) ? '1' : '0',
3610	Pdpe.u & PGM_PLXFLAGS_PERMANENT ? 'p' : '-',
3611	Pdpe.u & RT_BIT(11) ? '1' : '0',
3612	Pdpe.u & X86_PDPE_PG_MASK);
3613	if (cMaxDepth >= 1)
3614	{
3615	int rc2 = pgmR3DumpHierarchyHCPaePD(pVM, Pdpe.u & X86_PDPE_PG_MASK, u64Address + ((uint64_t)i << X86_PDPT_SHIFT),
3616	cr4, fLongMode, cMaxDepth - 1, pHlp);
3617	if (rc2 < rc && RT_SUCCESS(rc))
3618	rc = rc2;
3619	}
3620	}
3621	}
3622	return rc;
3623	}
3624
3625
3626	/**
3627	* Dumps a 32-bit shadow page table.
3628	*
3629	* @returns VBox status code (VINF_SUCCESS).
3630	* @param pVM The VM handle.
3631	* @param HCPhys The physical address of the table.
3632	* @param cr4 The CR4, PSE is currently used.
3633	* @param cMaxDepth The maxium depth.
3634	* @param pHlp Pointer to the output functions.
3635	*/
3636	static int pgmR3DumpHierarchyHcPaePML4(PVM pVM, RTHCPHYS HCPhys, uint32_t cr4, unsigned cMaxDepth, PCDBGFINFOHLP pHlp)
3637	{
3638	PX86PML4 pPML4 = (PX86PML4)MMPagePhys2Page(pVM, HCPhys);
3639	if (!pPML4)
3640	{
3641	pHlp->pfnPrintf(pHlp, "Page map level 4 at HCPhys=%RHp was not found in the page pool!\n", HCPhys);
3642	return VERR_INVALID_PARAMETER;
3643	}
3644
3645	int rc = VINF_SUCCESS;
3646	for (unsigned i = 0; i < RT_ELEMENTS(pPML4->a); i++)
3647	{
3648	X86PML4E Pml4e = pPML4->a[i];
3649	if (Pml4e.n.u1Present)
3650	{
3651	uint64_t u64Address = ((uint64_t)i << X86_PML4_SHIFT) \| (((uint64_t)i >> (X86_PML4_SHIFT - X86_PDPT_SHIFT - 1)) * 0xffff000000000000ULL);
3652	pHlp->pfnPrintf(pHlp, /P R S A D G WT CD AT NX 4M a p ? /
3653	"%016llx 0 \| P %c %c %c %c %c %s %s %s %s .. %c%c%c %016llx\n",
3654	u64Address,
3655	Pml4e.n.u1Write ? 'W' : 'R',
3656	Pml4e.n.u1User ? 'U' : 'S',
3657	Pml4e.n.u1Accessed ? 'A' : '-',
3658	Pml4e.n.u3Reserved & 1? '?' : '.', /* ignored */
3659	Pml4e.n.u3Reserved & 4? '!' : '.', /* mbz */
3660	Pml4e.n.u1WriteThru ? "WT" : "--",
3661	Pml4e.n.u1CacheDisable? "CD" : "--",
3662	Pml4e.n.u3Reserved & 2? "!" : "..",/* mbz */
3663	Pml4e.n.u1NoExecute ? "NX" : "--",
3664	Pml4e.u & RT_BIT(9) ? '1' : '0',
3665	Pml4e.u & PGM_PLXFLAGS_PERMANENT ? 'p' : '-',
3666	Pml4e.u & RT_BIT(11) ? '1' : '0',
3667	Pml4e.u & X86_PML4E_PG_MASK);
3668
3669	if (cMaxDepth >= 1)
3670	{
3671	int rc2 = pgmR3DumpHierarchyHCPaePDPT(pVM, Pml4e.u & X86_PML4E_PG_MASK, u64Address, cr4, true, cMaxDepth - 1, pHlp);
3672	if (rc2 < rc && RT_SUCCESS(rc))
3673	rc = rc2;
3674	}
3675	}
3676	}
3677	return rc;
3678	}
3679
3680
3681	/**
3682	* Dumps a 32-bit shadow page table.
3683	*
3684	* @returns VBox status code (VINF_SUCCESS).
3685	* @param pVM The VM handle.
3686	* @param pPT Pointer to the page table.
3687	* @param u32Address The virtual address this table starts at.
3688	* @param pHlp Pointer to the output functions.
3689	*/
3690	int pgmR3DumpHierarchyHC32BitPT(PVM pVM, PX86PT pPT, uint32_t u32Address, PCDBGFINFOHLP pHlp)
3691	{
3692	for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
3693	{
3694	X86PTE Pte = pPT->a[i];
3695	if (Pte.n.u1Present)
3696	{
3697	pHlp->pfnPrintf(pHlp, /P R S A D G WT CD AT NX 4M a m d /
3698	"%08x 1 \| P %c %c %c %c %c %s %s %s .. 4K %c%c%c %08x\n",
3699	u32Address + (i << X86_PT_SHIFT),
3700	Pte.n.u1Write ? 'W' : 'R',
3701	Pte.n.u1User ? 'U' : 'S',
3702	Pte.n.u1Accessed ? 'A' : '-',
3703	Pte.n.u1Dirty ? 'D' : '-',
3704	Pte.n.u1Global ? 'G' : '-',
3705	Pte.n.u1WriteThru ? "WT" : "--",
3706	Pte.n.u1CacheDisable? "CD" : "--",
3707	Pte.n.u1PAT ? "AT" : "--",
3708	Pte.u & PGM_PTFLAGS_TRACK_DIRTY ? 'd' : '-',
3709	Pte.u & RT_BIT(10) ? '1' : '0',
3710	Pte.u & PGM_PTFLAGS_CSAM_VALIDATED ? 'v' : '-',
3711	Pte.u & X86_PDE_PG_MASK);
3712	}
3713	}
3714	return VINF_SUCCESS;
3715	}
3716
3717
3718	/**
3719	* Dumps a 32-bit shadow page directory and page tables.
3720	*
3721	* @returns VBox status code (VINF_SUCCESS).
3722	* @param pVM The VM handle.
3723	* @param cr3 The root of the hierarchy.
3724	* @param cr4 The CR4, PSE is currently used.
3725	* @param cMaxDepth How deep into the hierarchy the dumper should go.
3726	* @param pHlp Pointer to the output functions.
3727	*/
3728	int pgmR3DumpHierarchyHC32BitPD(PVM pVM, uint32_t cr3, uint32_t cr4, unsigned cMaxDepth, PCDBGFINFOHLP pHlp)
3729	{
3730	PX86PD pPD = (PX86PD)MMPagePhys2Page(pVM, cr3 & X86_CR3_PAGE_MASK);
3731	if (!pPD)
3732	{
3733	pHlp->pfnPrintf(pHlp, "Page directory at %#x was not found in the page pool!\n", cr3 & X86_CR3_PAGE_MASK);
3734	return VERR_INVALID_PARAMETER;
3735	}
3736
3737	int rc = VINF_SUCCESS;
3738	for (unsigned i = 0; i < RT_ELEMENTS(pPD->a); i++)
3739	{
3740	X86PDE Pde = pPD->a[i];
3741	if (Pde.n.u1Present)
3742	{
3743	const uint32_t u32Address = i << X86_PD_SHIFT;
3744	if ((cr4 & X86_CR4_PSE) && Pde.b.u1Size)
3745	pHlp->pfnPrintf(pHlp, /P R S A D G WT CD AT NX 4M a m d /
3746	"%08x 0 \| P %c %c %c %c %c %s %s %s .. 4M %c%c%c %08x\n",
3747	u32Address,
3748	Pde.b.u1Write ? 'W' : 'R',
3749	Pde.b.u1User ? 'U' : 'S',
3750	Pde.b.u1Accessed ? 'A' : '-',
3751	Pde.b.u1Dirty ? 'D' : '-',
3752	Pde.b.u1Global ? 'G' : '-',
3753	Pde.b.u1WriteThru ? "WT" : "--",
3754	Pde.b.u1CacheDisable? "CD" : "--",
3755	Pde.b.u1PAT ? "AT" : "--",
3756	Pde.u & RT_BIT_64(9) ? '1' : '0',
3757	Pde.u & PGM_PDFLAGS_MAPPING ? 'm' : '-',
3758	Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
3759	Pde.u & X86_PDE4M_PG_MASK);
3760	else
3761	{
3762	pHlp->pfnPrintf(pHlp, /P R S A D G WT CD AT NX 4M a m d /
3763	"%08x 0 \| P %c %c %c %c %c %s %s .. .. 4K %c%c%c %08x\n",
3764	u32Address,
3765	Pde.n.u1Write ? 'W' : 'R',
3766	Pde.n.u1User ? 'U' : 'S',
3767	Pde.n.u1Accessed ? 'A' : '-',
3768	Pde.n.u1Reserved0 ? '?' : '.', /* ignored */
3769	Pde.n.u1Reserved1 ? '?' : '.', /* ignored */
3770	Pde.n.u1WriteThru ? "WT" : "--",
3771	Pde.n.u1CacheDisable? "CD" : "--",
3772	Pde.u & RT_BIT_64(9) ? '1' : '0',
3773	Pde.u & PGM_PDFLAGS_MAPPING ? 'm' : '-',
3774	Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
3775	Pde.u & X86_PDE_PG_MASK);
3776	if (cMaxDepth >= 1)
3777	{
3778	/** @todo what about using the page pool for mapping PTs? */
3779	RTHCPHYS HCPhys = Pde.u & X86_PDE_PG_MASK;
3780	PX86PT pPT = NULL;
3781	if (!(Pde.u & PGM_PDFLAGS_MAPPING))
3782	pPT = (PX86PT)MMPagePhys2Page(pVM, HCPhys);
3783	else
3784	{
3785	for (PPGMMAPPING pMap = pVM->pgm.s.pMappingsR3; pMap; pMap = pMap->pNextR3)
3786	if (u32Address - pMap->GCPtr < pMap->cb)
3787	{
3788	int iPDE = (u32Address - pMap->GCPtr) >> X86_PD_SHIFT;
3789	if (pMap->aPTs[iPDE].HCPhysPT != HCPhys)
3790	pHlp->pfnPrintf(pHlp, "%08x error! Mapping error! PT %d has HCPhysPT=%RHp not %RHp is in the PD.\n",
3791	u32Address, iPDE, pMap->aPTs[iPDE].HCPhysPT, HCPhys);
3792	pPT = pMap->aPTs[iPDE].pPTR3;
3793	}
3794	}
3795	int rc2 = VERR_INVALID_PARAMETER;
3796	if (pPT)
3797	rc2 = pgmR3DumpHierarchyHC32BitPT(pVM, pPT, u32Address, pHlp);
3798	else
3799	pHlp->pfnPrintf(pHlp, "%08x error! Page table at %#x was not found in the page pool!\n", u32Address, HCPhys);
3800	if (rc2 < rc && RT_SUCCESS(rc))
3801	rc = rc2;
3802	}
3803	}
3804	}
3805	}
3806
3807	return rc;
3808	}
3809
3810
3811	/**
3812	* Dumps a 32-bit shadow page table.
3813	*
3814	* @returns VBox status code (VINF_SUCCESS).
3815	* @param pVM The VM handle.
3816	* @param pPT Pointer to the page table.
3817	* @param u32Address The virtual address this table starts at.
3818	* @param PhysSearch Address to search for.
3819	*/
3820	int pgmR3DumpHierarchyGC32BitPT(PVM pVM, PX86PT pPT, uint32_t u32Address, RTGCPHYS PhysSearch)
3821	{
3822	for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
3823	{
3824	X86PTE Pte = pPT->a[i];
3825	if (Pte.n.u1Present)
3826	{
3827	Log(( /P R S A D G WT CD AT NX 4M a m d /
3828	"%08x 1 \| P %c %c %c %c %c %s %s %s .. 4K %c%c%c %08x\n",
3829	u32Address + (i << X86_PT_SHIFT),
3830	Pte.n.u1Write ? 'W' : 'R',
3831	Pte.n.u1User ? 'U' : 'S',
3832	Pte.n.u1Accessed ? 'A' : '-',
3833	Pte.n.u1Dirty ? 'D' : '-',
3834	Pte.n.u1Global ? 'G' : '-',
3835	Pte.n.u1WriteThru ? "WT" : "--",
3836	Pte.n.u1CacheDisable? "CD" : "--",
3837	Pte.n.u1PAT ? "AT" : "--",
3838	Pte.u & PGM_PTFLAGS_TRACK_DIRTY ? 'd' : '-',
3839	Pte.u & RT_BIT(10) ? '1' : '0',
3840	Pte.u & PGM_PTFLAGS_CSAM_VALIDATED ? 'v' : '-',
3841	Pte.u & X86_PDE_PG_MASK));
3842
3843	if ((Pte.u & X86_PDE_PG_MASK) == PhysSearch)
3844	{
3845	uint64_t fPageShw = 0;
3846	RTHCPHYS pPhysHC = 0;
3847
3848	/** @todo SMP support!! */
3849	PGMShwGetPage(&pVM->aCpus[0], (RTGCPTR)(u32Address + (i << X86_PT_SHIFT)), &fPageShw, &pPhysHC);
3850	Log(("Found %RGp at %RGv -> flags=%llx\n", PhysSearch, (RTGCPTR)(u32Address + (i << X86_PT_SHIFT)), fPageShw));
3851	}
3852	}
3853	}
3854	return VINF_SUCCESS;
3855	}
3856
3857
3858	/**
3859	* Dumps a 32-bit guest page directory and page tables.
3860	*
3861	* @returns VBox status code (VINF_SUCCESS).
3862	* @param pVM The VM handle.
3863	* @param cr3 The root of the hierarchy.
3864	* @param cr4 The CR4, PSE is currently used.
3865	* @param PhysSearch Address to search for.
3866	*/
3867	VMMR3DECL(int) PGMR3DumpHierarchyGC(PVM pVM, uint64_t cr3, uint64_t cr4, RTGCPHYS PhysSearch)
3868	{
3869	bool fLongMode = false;
3870	const unsigned cch = fLongMode ? 16 : 8; NOREF(cch);
3871	PX86PD pPD = 0;
3872
3873	int rc = PGM_GCPHYS_2_PTR(pVM, cr3 & X86_CR3_PAGE_MASK, &pPD);
3874	if (RT_FAILURE(rc) \|\| !pPD)
3875	{
3876	Log(("Page directory at %#x was not found in the page pool!\n", cr3 & X86_CR3_PAGE_MASK));
3877	return VERR_INVALID_PARAMETER;
3878	}
3879
3880	Log(("cr3=%08x cr4=%08x%s\n"
3881	"%-*s P - Present\n"
3882	"%-*s \| R/W - Read (0) / Write (1)\n"
3883	"%-*s \| \| U/S - User (1) / Supervisor (0)\n"
3884	"%-*s \| \| \| A - Accessed\n"
3885	"%-*s \| \| \| \| D - Dirty\n"
3886	"%-*s \| \| \| \| \| G - Global\n"
3887	"%-*s \| \| \| \| \| \| WT - Write thru\n"
3888	"%-*s \| \| \| \| \| \| \| CD - Cache disable\n"
3889	"%-*s \| \| \| \| \| \| \| \| AT - Attribute table (PAT)\n"
3890	"%-*s \| \| \| \| \| \| \| \| \| NX - No execute (K8)\n"
3891	"%-*s \| \| \| \| \| \| \| \| \| \| 4K/4M/2M - Page size.\n"
3892	"%-*s \| \| \| \| \| \| \| \| \| \| \| AVL - a=allocated; m=mapping; d=track dirty;\n"
3893	"%-*s \| \| \| \| \| \| \| \| \| \| \| \| p=permanent; v=validated;\n"
3894	"%-*s Level \| \| \| \| \| \| \| \| \| \| \| \| Page\n"
3895	/* xxxx n **** P R S A D G WT CD AT NX 4M AVL xxxxxxxxxxxxx
3896	- W U - - - -- -- -- -- -- 010 */
3897	, cr3, cr4, fLongMode ? " Long Mode" : "",
3898	cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "",
3899	cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "Address"));
3900
3901	for (unsigned i = 0; i < RT_ELEMENTS(pPD->a); i++)
3902	{
3903	X86PDE Pde = pPD->a[i];
3904	if (Pde.n.u1Present)
3905	{
3906	const uint32_t u32Address = i << X86_PD_SHIFT;
3907
3908	if ((cr4 & X86_CR4_PSE) && Pde.b.u1Size)
3909	Log(( /P R S A D G WT CD AT NX 4M a m d /
3910	"%08x 0 \| P %c %c %c %c %c %s %s %s .. 4M %c%c%c %08x\n",
3911	u32Address,
3912	Pde.b.u1Write ? 'W' : 'R',
3913	Pde.b.u1User ? 'U' : 'S',
3914	Pde.b.u1Accessed ? 'A' : '-',
3915	Pde.b.u1Dirty ? 'D' : '-',
3916	Pde.b.u1Global ? 'G' : '-',
3917	Pde.b.u1WriteThru ? "WT" : "--",
3918	Pde.b.u1CacheDisable? "CD" : "--",
3919	Pde.b.u1PAT ? "AT" : "--",
3920	Pde.u & RT_BIT(9) ? '1' : '0',
3921	Pde.u & RT_BIT(10) ? '1' : '0',
3922	Pde.u & RT_BIT(11) ? '1' : '0',
3923	pgmGstGet4MBPhysPage(&pVM->pgm.s, Pde)));
3924	/** @todo PhysSearch */
3925	else
3926	{
3927	Log(( /P R S A D G WT CD AT NX 4M a m d /
3928	"%08x 0 \| P %c %c %c %c %c %s %s .. .. 4K %c%c%c %08x\n",
3929	u32Address,
3930	Pde.n.u1Write ? 'W' : 'R',
3931	Pde.n.u1User ? 'U' : 'S',
3932	Pde.n.u1Accessed ? 'A' : '-',
3933	Pde.n.u1Reserved0 ? '?' : '.', /* ignored */
3934	Pde.n.u1Reserved1 ? '?' : '.', /* ignored */
3935	Pde.n.u1WriteThru ? "WT" : "--",
3936	Pde.n.u1CacheDisable? "CD" : "--",
3937	Pde.u & RT_BIT(9) ? '1' : '0',
3938	Pde.u & RT_BIT(10) ? '1' : '0',
3939	Pde.u & RT_BIT(11) ? '1' : '0',
3940	Pde.u & X86_PDE_PG_MASK));
3941	////if (cMaxDepth >= 1)
3942	{
3943	/** @todo what about using the page pool for mapping PTs? */
3944	RTGCPHYS GCPhys = Pde.u & X86_PDE_PG_MASK;
3945	PX86PT pPT = NULL;
3946
3947	rc = PGM_GCPHYS_2_PTR(pVM, GCPhys, &pPT);
3948
3949	int rc2 = VERR_INVALID_PARAMETER;
3950	if (pPT)
3951	rc2 = pgmR3DumpHierarchyGC32BitPT(pVM, pPT, u32Address, PhysSearch);
3952	else
3953	Log(("%08x error! Page table at %#x was not found in the page pool!\n", u32Address, GCPhys));
3954	if (rc2 < rc && RT_SUCCESS(rc))
3955	rc = rc2;
3956	}
3957	}
3958	}
3959	}
3960
3961	return rc;
3962	}
3963
3964
3965	/**
3966	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3967	*
3968	* @returns VBox status code (VINF_SUCCESS).
3969	* @param pVM The VM handle.
3970	* @param cr3 The root of the hierarchy.
3971	* @param cr4 The cr4, only PAE and PSE is currently used.
3972	* @param fLongMode Set if long mode, false if not long mode.
3973	* @param cMaxDepth Number of levels to dump.
3974	* @param pHlp Pointer to the output functions.
3975	*/
3976	VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint64_t cr3, uint64_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp)
3977	{
3978	if (!pHlp)
3979	pHlp = DBGFR3InfoLogHlp();
3980	if (!cMaxDepth)
3981	return VINF_SUCCESS;
3982	const unsigned cch = fLongMode ? 16 : 8;
3983	pHlp->pfnPrintf(pHlp,
3984	"cr3=%08x cr4=%08x%s\n"
3985	"%-*s P - Present\n"
3986	"%-*s \| R/W - Read (0) / Write (1)\n"
3987	"%-*s \| \| U/S - User (1) / Supervisor (0)\n"
3988	"%-*s \| \| \| A - Accessed\n"
3989	"%-*s \| \| \| \| D - Dirty\n"
3990	"%-*s \| \| \| \| \| G - Global\n"
3991	"%-*s \| \| \| \| \| \| WT - Write thru\n"
3992	"%-*s \| \| \| \| \| \| \| CD - Cache disable\n"
3993	"%-*s \| \| \| \| \| \| \| \| AT - Attribute table (PAT)\n"
3994	"%-*s \| \| \| \| \| \| \| \| \| NX - No execute (K8)\n"
3995	"%-*s \| \| \| \| \| \| \| \| \| \| 4K/4M/2M - Page size.\n"
3996	"%-*s \| \| \| \| \| \| \| \| \| \| \| AVL - a=allocated; m=mapping; d=track dirty;\n"
3997	"%-*s \| \| \| \| \| \| \| \| \| \| \| \| p=permanent; v=validated;\n"
3998	"%-*s Level \| \| \| \| \| \| \| \| \| \| \| \| Page\n"
3999	/* xxxx n **** P R S A D G WT CD AT NX 4M AVL xxxxxxxxxxxxx
4000	- W U - - - -- -- -- -- -- 010 */
4001	, cr3, cr4, fLongMode ? " Long Mode" : "",
4002	cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "",
4003	cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "Address");
4004	if (cr4 & X86_CR4_PAE)
4005	{
4006	if (fLongMode)
4007	return pgmR3DumpHierarchyHcPaePML4(pVM, cr3 & X86_CR3_PAGE_MASK, cr4, cMaxDepth, pHlp);
4008	return pgmR3DumpHierarchyHCPaePDPT(pVM, cr3 & X86_CR3_PAE_PAGE_MASK, 0, cr4, false, cMaxDepth, pHlp);
4009	}
4010	return pgmR3DumpHierarchyHC32BitPD(pVM, cr3 & X86_CR3_PAGE_MASK, cr4, cMaxDepth, pHlp);
4011	}
4012
4013	#ifdef VBOX_WITH_DEBUGGER
4014
4015	/**
4016	* The '.pgmram' command.
4017	*
4018	* @returns VBox status.
4019	* @param pCmd Pointer to the command descriptor (as registered).
4020	* @param pCmdHlp Pointer to command helper functions.
4021	* @param pVM Pointer to the current VM (if any).
4022	* @param paArgs Pointer to (readonly) array of arguments.
4023	* @param cArgs Number of arguments in the array.
4024	*/
4025	static DECLCALLBACK(int) pgmR3CmdRam(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
4026	{
4027	/*
4028	* Validate input.
4029	*/
4030	if (!pVM)
4031	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
4032	if (!pVM->pgm.s.pRamRangesRC)
4033	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Sorry, no Ram is registered.\n");
4034
4035	/*
4036	* Dump the ranges.
4037	*/
4038	int rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL, "From - To (incl) pvHC\n");
4039	PPGMRAMRANGE pRam;
4040	for (pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
4041	{
4042	rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL,
4043	"%RGp - %RGp %p\n",
4044	pRam->GCPhys, pRam->GCPhysLast, pRam->pvR3);
4045	if (RT_FAILURE(rc))
4046	return rc;
4047	}
4048
4049	return VINF_SUCCESS;
4050	}
4051
4052
4053	/**
4054	* The '.pgmerror' and '.pgmerroroff' commands.
4055	*
4056	* @returns VBox status.
4057	* @param pCmd Pointer to the command descriptor (as registered).
4058	* @param pCmdHlp Pointer to command helper functions.
4059	* @param pVM Pointer to the current VM (if any).
4060	* @param paArgs Pointer to (readonly) array of arguments.
4061	* @param cArgs Number of arguments in the array.
4062	*/
4063	static DECLCALLBACK(int) pgmR3CmdError(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
4064	{
4065	/*
4066	* Validate input.
4067	*/
4068	if (!pVM)
4069	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
4070	AssertReturn(cArgs == 0 \|\| (cArgs == 1 && paArgs[0].enmType == DBGCVAR_TYPE_STRING),
4071	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: Hit bug in the parser.\n"));
4072
4073	if (!cArgs)
4074	{
4075	/*
4076	* Print the list of error injection locations with status.
4077	*/
4078	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "PGM error inject locations:\n");
4079	pCmdHlp->pfnPrintf(pCmdHlp, NULL, " handy - %RTbool\n", pVM->pgm.s.fErrInjHandyPages);
4080	}
4081	else
4082	{
4083
4084	/*
4085	* String switch on where to inject the error.
4086	*/
4087	bool const fNewState = !strcmp(pCmd->pszCmd, "pgmerror");
4088	const char *pszWhere = paArgs[0].u.pszString;
4089	if (!strcmp(pszWhere, "handy"))
4090	ASMAtomicWriteBool(&pVM->pgm.s.fErrInjHandyPages, fNewState);
4091	else
4092	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: Invalid 'where' value: %s.\n", pszWhere);
4093	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "done\n");
4094	}
4095	return VINF_SUCCESS;
4096	}
4097
4098
4099	/**
4100	* The '.pgmsync' command.
4101	*
4102	* @returns VBox status.
4103	* @param pCmd Pointer to the command descriptor (as registered).
4104	* @param pCmdHlp Pointer to command helper functions.
4105	* @param pVM Pointer to the current VM (if any).
4106	* @param paArgs Pointer to (readonly) array of arguments.
4107	* @param cArgs Number of arguments in the array.
4108	*/
4109	static DECLCALLBACK(int) pgmR3CmdSync(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
4110	{
4111	/** @todo SMP support */
4112	PVMCPU pVCpu = &pVM->aCpus[0];
4113
4114	/*
4115	* Validate input.
4116	*/
4117	if (!pVM)
4118	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
4119
4120	/*
4121	* Force page directory sync.
4122	*/
4123	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
4124
4125	int rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Forcing page directory sync.\n");
4126	if (RT_FAILURE(rc))
4127	return rc;
4128
4129	return VINF_SUCCESS;
4130	}
4131
4132
4133	#ifdef VBOX_STRICT
4134	/**
4135	* The '.pgmassertcr3' command.
4136	*
4137	* @returns VBox status.
4138	* @param pCmd Pointer to the command descriptor (as registered).
4139	* @param pCmdHlp Pointer to command helper functions.
4140	* @param pVM Pointer to the current VM (if any).
4141	* @param paArgs Pointer to (readonly) array of arguments.
4142	* @param cArgs Number of arguments in the array.
4143	*/
4144	static DECLCALLBACK(int) pgmR3CmdAssertCR3(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
4145	{
4146	/** @todo SMP support!! */
4147	PVMCPU pVCpu = &pVM->aCpus[0];
4148
4149	/*
4150	* Validate input.
4151	*/
4152	if (!pVM)
4153	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
4154
4155	int rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Checking shadow CR3 page tables for consistency.\n");
4156	if (RT_FAILURE(rc))
4157	return rc;
4158
4159	PGMAssertCR3(pVM, pVCpu, CPUMGetGuestCR3(pVCpu), CPUMGetGuestCR4(pVCpu));
4160
4161	return VINF_SUCCESS;
4162	}
4163	#endif /* VBOX_STRICT */
4164
4165
4166	/**
4167	* The '.pgmsyncalways' command.
4168	*
4169	* @returns VBox status.
4170	* @param pCmd Pointer to the command descriptor (as registered).
4171	* @param pCmdHlp Pointer to command helper functions.
4172	* @param pVM Pointer to the current VM (if any).
4173	* @param paArgs Pointer to (readonly) array of arguments.
4174	* @param cArgs Number of arguments in the array.
4175	*/
4176	static DECLCALLBACK(int) pgmR3CmdSyncAlways(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
4177	{
4178	/** @todo SMP support!! */
4179	PVMCPU pVCpu = &pVM->aCpus[0];
4180
4181	/*
4182	* Validate input.
4183	*/
4184	if (!pVM)
4185	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
4186
4187	/*
4188	* Force page directory sync.
4189	*/
4190	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS)
4191	{
4192	ASMAtomicAndU32(&pVCpu->pgm.s.fSyncFlags, ~PGM_SYNC_ALWAYS);
4193	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Disabled permanent forced page directory syncing.\n");
4194	}
4195	else
4196	{
4197	ASMAtomicOrU32(&pVCpu->pgm.s.fSyncFlags, PGM_SYNC_ALWAYS);
4198	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
4199	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Enabled permanent forced page directory syncing.\n");
4200	}
4201	}
4202
4203
4204	/**
4205	* The '.pgmsyncalways' command.
4206	*
4207	* @returns VBox status.
4208	* @param pCmd Pointer to the command descriptor (as registered).
4209	* @param pCmdHlp Pointer to command helper functions.
4210	* @param pVM Pointer to the current VM (if any).
4211	* @param paArgs Pointer to (readonly) array of arguments.
4212	* @param cArgs Number of arguments in the array.
4213	*/
4214	static DECLCALLBACK(int) pgmR3CmdPhysToFile(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
4215	{
4216	/*
4217	* Validate input.
4218	*/
4219	if (!pVM)
4220	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
4221	if ( cArgs < 1
4222	\|\| cArgs > 2
4223	\|\| paArgs[0].enmType != DBGCVAR_TYPE_STRING
4224	\|\| ( cArgs > 1
4225	&& paArgs[1].enmType != DBGCVAR_TYPE_STRING))
4226	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: parser error, invalid arguments.\n");
4227	if ( cArgs >= 2
4228	&& strcmp(paArgs[1].u.pszString, "nozero"))
4229	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: Invalid 2nd argument '%s', must be 'nozero'.\n", paArgs[1].u.pszString);
4230	bool fIncZeroPgs = cArgs < 2;
4231
4232	/*
4233	* Open the output file and get the ram parameters.
4234	*/
4235	RTFILE hFile;
4236	int rc = RTFileOpen(&hFile, paArgs[0].u.pszString, RTFILE_O_WRITE \| RTFILE_O_CREATE_REPLACE \| RTFILE_O_DENY_WRITE);
4237	if (RT_FAILURE(rc))
4238	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: RTFileOpen(,'%s',) -> %Rrc.\n", paArgs[0].u.pszString, rc);
4239
4240	uint32_t cbRamHole = 0;
4241	CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "RamHoleSize", &cbRamHole, MM_RAM_HOLE_SIZE_DEFAULT);
4242	uint64_t cbRam = 0;
4243	CFGMR3QueryU64Def(CFGMR3GetRoot(pVM), "RamSize", &cbRam, 0);
4244	RTGCPHYS GCPhysEnd = cbRam + cbRamHole;
4245
4246	/*
4247	* Dump the physical memory, page by page.
4248	*/
4249	RTGCPHYS GCPhys = 0;
4250	char abZeroPg[PAGE_SIZE];
4251	RT_ZERO(abZeroPg);
4252
4253	pgmLock(pVM);
4254	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
4255	pRam && pRam->GCPhys < GCPhysEnd && RT_SUCCESS(rc);
4256	pRam = pRam->pNextR3)
4257	{
4258	/* fill the gap */
4259	if (pRam->GCPhys > GCPhys && fIncZeroPgs)
4260	{
4261	while (pRam->GCPhys > GCPhys && RT_SUCCESS(rc))
4262	{
4263	rc = RTFileWrite(hFile, abZeroPg, PAGE_SIZE, NULL);
4264	GCPhys += PAGE_SIZE;
4265	}
4266	}
4267
4268	PCPGMPAGE pPage = &pRam->aPages[0];
4269	while (GCPhys < pRam->GCPhysLast && RT_SUCCESS(rc))
4270	{
4271	if (PGM_PAGE_IS_ZERO(pPage))
4272	{
4273	if (fIncZeroPgs)
4274	{
4275	rc = RTFileWrite(hFile, abZeroPg, PAGE_SIZE, NULL);
4276	if (RT_FAILURE(rc))
4277	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: RTFileWrite -> %Rrc at GCPhys=%RGp.\n", rc, GCPhys);
4278	}
4279	}
4280	else
4281	{
4282	switch (PGM_PAGE_GET_TYPE(pPage))
4283	{
4284	case PGMPAGETYPE_RAM:
4285	case PGMPAGETYPE_ROM_SHADOW: /* trouble?? */
4286	case PGMPAGETYPE_ROM:
4287	case PGMPAGETYPE_MMIO2:
4288	{
4289	void const *pvPage;
4290	PGMPAGEMAPLOCK Lock;
4291	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvPage, &Lock);
4292	if (RT_SUCCESS(rc))
4293	{
4294	rc = RTFileWrite(hFile, pvPage, PAGE_SIZE, NULL);
4295	PGMPhysReleasePageMappingLock(pVM, &Lock);
4296	if (RT_FAILURE(rc))
4297	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: RTFileWrite -> %Rrc at GCPhys=%RGp.\n", rc, GCPhys);
4298	}
4299	else
4300	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: PGMPhysGCPhys2CCPtrReadOnly -> %Rrc at GCPhys=%RGp.\n", rc, GCPhys);
4301	break;
4302	}
4303
4304	default:
4305	AssertFailed();
4306	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
4307	case PGMPAGETYPE_MMIO:
4308	if (fIncZeroPgs)
4309	{
4310	rc = RTFileWrite(hFile, abZeroPg, PAGE_SIZE, NULL);
4311	if (RT_FAILURE(rc))
4312	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: RTFileWrite -> %Rrc at GCPhys=%RGp.\n", rc, GCPhys);
4313	}
4314	break;
4315	}
4316	}
4317
4318
4319	/* advance */
4320	GCPhys += PAGE_SIZE;
4321	pPage++;
4322	}
4323	}
4324	pgmUnlock(pVM);
4325
4326	RTFileClose(hFile);
4327	if (RT_SUCCESS(rc))
4328	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Successfully saved physical memory to '%s'.\n", paArgs[0].u.pszString);
4329	return VINF_SUCCESS;
4330	}
4331
4332	#endif /* VBOX_WITH_DEBUGGER */
4333
4334	/**
4335	* pvUser argument of the pgmR3CheckIntegrity*Node callbacks.
4336	*/
4337	typedef struct PGMCHECKINTARGS
4338	{
4339	bool fLeftToRight; /*< true: left-to-right; false: right-to-left. /
4340	PPGMPHYSHANDLER pPrevPhys;
4341	PPGMVIRTHANDLER pPrevVirt;
4342	PPGMPHYS2VIRTHANDLER pPrevPhys2Virt;
4343	PVM pVM;
4344	} PGMCHECKINTARGS, *PPGMCHECKINTARGS;
4345
4346	/**
4347	* Validate a node in the physical handler tree.
4348	*
4349	* @returns 0 on if ok, other wise 1.
4350	* @param pNode The handler node.
4351	* @param pvUser pVM.
4352	*/
4353	static DECLCALLBACK(int) pgmR3CheckIntegrityPhysHandlerNode(PAVLROGCPHYSNODECORE pNode, void *pvUser)
4354	{
4355	PPGMCHECKINTARGS pArgs = (PPGMCHECKINTARGS)pvUser;
4356	PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)pNode;
4357	AssertReleaseReturn(!((uintptr_t)pCur & 7), 1);
4358	AssertReleaseMsg(pCur->Core.Key <= pCur->Core.KeyLast,("pCur=%p %RGp-%RGp %s\n", pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
4359	AssertReleaseMsg( !pArgs->pPrevPhys
4360	\|\| (pArgs->fLeftToRight ? pArgs->pPrevPhys->Core.KeyLast < pCur->Core.Key : pArgs->pPrevPhys->Core.KeyLast > pCur->Core.Key),
4361	("pPrevPhys=%p %RGp-%RGp %s\n"
4362	" pCur=%p %RGp-%RGp %s\n",
4363	pArgs->pPrevPhys, pArgs->pPrevPhys->Core.Key, pArgs->pPrevPhys->Core.KeyLast, pArgs->pPrevPhys->pszDesc,
4364	pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
4365	pArgs->pPrevPhys = pCur;
4366	return 0;
4367	}
4368
4369
4370	/**
4371	* Validate a node in the virtual handler tree.
4372	*
4373	* @returns 0 on if ok, other wise 1.
4374	* @param pNode The handler node.
4375	* @param pvUser pVM.
4376	*/
4377	static DECLCALLBACK(int) pgmR3CheckIntegrityVirtHandlerNode(PAVLROGCPTRNODECORE pNode, void *pvUser)
4378	{
4379	PPGMCHECKINTARGS pArgs = (PPGMCHECKINTARGS)pvUser;
4380	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)pNode;
4381	AssertReleaseReturn(!((uintptr_t)pCur & 7), 1);
4382	AssertReleaseMsg(pCur->Core.Key <= pCur->Core.KeyLast,("pCur=%p %RGv-%RGv %s\n", pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
4383	AssertReleaseMsg( !pArgs->pPrevVirt
4384	\|\| (pArgs->fLeftToRight ? pArgs->pPrevVirt->Core.KeyLast < pCur->Core.Key : pArgs->pPrevVirt->Core.KeyLast > pCur->Core.Key),
4385	("pPrevVirt=%p %RGv-%RGv %s\n"
4386	" pCur=%p %RGv-%RGv %s\n",
4387	pArgs->pPrevVirt, pArgs->pPrevVirt->Core.Key, pArgs->pPrevVirt->Core.KeyLast, pArgs->pPrevVirt->pszDesc,
4388	pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
4389	for (unsigned iPage = 0; iPage < pCur->cPages; iPage++)
4390	{
4391	AssertReleaseMsg(pCur->aPhysToVirt[iPage].offVirtHandler == -RT_OFFSETOF(PGMVIRTHANDLER, aPhysToVirt[iPage]),
4392	("pCur=%p %RGv-%RGv %s\n"
4393	"iPage=%d offVirtHandle=%#x expected %#x\n",
4394	pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc,
4395	iPage, pCur->aPhysToVirt[iPage].offVirtHandler, -RT_OFFSETOF(PGMVIRTHANDLER, aPhysToVirt[iPage])));
4396	}
4397	pArgs->pPrevVirt = pCur;
4398	return 0;
4399	}
4400
4401
4402	/**
4403	* Validate a node in the virtual handler tree.
4404	*
4405	* @returns 0 on if ok, other wise 1.
4406	* @param pNode The handler node.
4407	* @param pvUser pVM.
4408	*/
4409	static DECLCALLBACK(int) pgmR3CheckIntegrityPhysToVirtHandlerNode(PAVLROGCPHYSNODECORE pNode, void *pvUser)
4410	{
4411	PPGMCHECKINTARGS pArgs = (PPGMCHECKINTARGS)pvUser;
4412	PPGMPHYS2VIRTHANDLER pCur = (PPGMPHYS2VIRTHANDLER)pNode;
4413	AssertReleaseMsgReturn(!((uintptr_t)pCur & 3), ("\n"), 1);
4414	AssertReleaseMsgReturn(!(pCur->offVirtHandler & 3), ("\n"), 1);
4415	AssertReleaseMsg(pCur->Core.Key <= pCur->Core.KeyLast,("pCur=%p %RGp-%RGp\n", pCur, pCur->Core.Key, pCur->Core.KeyLast));
4416	AssertReleaseMsg( !pArgs->pPrevPhys2Virt
4417	\|\| (pArgs->fLeftToRight ? pArgs->pPrevPhys2Virt->Core.KeyLast < pCur->Core.Key : pArgs->pPrevPhys2Virt->Core.KeyLast > pCur->Core.Key),
4418	("pPrevPhys2Virt=%p %RGp-%RGp\n"
4419	" pCur=%p %RGp-%RGp\n",
4420	pArgs->pPrevPhys2Virt, pArgs->pPrevPhys2Virt->Core.Key, pArgs->pPrevPhys2Virt->Core.KeyLast,
4421	pCur, pCur->Core.Key, pCur->Core.KeyLast));
4422	AssertReleaseMsg( !pArgs->pPrevPhys2Virt
4423	\|\| (pArgs->fLeftToRight ? pArgs->pPrevPhys2Virt->Core.KeyLast < pCur->Core.Key : pArgs->pPrevPhys2Virt->Core.KeyLast > pCur->Core.Key),
4424	("pPrevPhys2Virt=%p %RGp-%RGp\n"
4425	" pCur=%p %RGp-%RGp\n",
4426	pArgs->pPrevPhys2Virt, pArgs->pPrevPhys2Virt->Core.Key, pArgs->pPrevPhys2Virt->Core.KeyLast,
4427	pCur, pCur->Core.Key, pCur->Core.KeyLast));
4428	AssertReleaseMsg((pCur->offNextAlias & (PGMPHYS2VIRTHANDLER_IN_TREE \| PGMPHYS2VIRTHANDLER_IS_HEAD)) == (PGMPHYS2VIRTHANDLER_IN_TREE \| PGMPHYS2VIRTHANDLER_IS_HEAD),
4429	("pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4430	pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias));
4431	if (pCur->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK)
4432	{
4433	PPGMPHYS2VIRTHANDLER pCur2 = pCur;
4434	for (;;)
4435	{
4436	pCur2 = (PPGMPHYS2VIRTHANDLER)((intptr_t)pCur + (pCur->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4437	AssertReleaseMsg(pCur2 != pCur,
4438	(" pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4439	pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias));
4440	AssertReleaseMsg((pCur2->offNextAlias & (PGMPHYS2VIRTHANDLER_IN_TREE \| PGMPHYS2VIRTHANDLER_IS_HEAD)) == PGMPHYS2VIRTHANDLER_IN_TREE,
4441	(" pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
4442	"pCur2=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4443	pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias,
4444	pCur2, pCur2->Core.Key, pCur2->Core.KeyLast, pCur2->offVirtHandler, pCur2->offNextAlias));
4445	AssertReleaseMsg((pCur2->Core.Key ^ pCur->Core.Key) < PAGE_SIZE,
4446	(" pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
4447	"pCur2=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4448	pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias,
4449	pCur2, pCur2->Core.Key, pCur2->Core.KeyLast, pCur2->offVirtHandler, pCur2->offNextAlias));
4450	AssertReleaseMsg((pCur2->Core.KeyLast ^ pCur->Core.KeyLast) < PAGE_SIZE,
4451	(" pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
4452	"pCur2=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4453	pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias,
4454	pCur2, pCur2->Core.Key, pCur2->Core.KeyLast, pCur2->offVirtHandler, pCur2->offNextAlias));
4455	if (!(pCur2->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK))
4456	break;
4457	}
4458	}
4459
4460	pArgs->pPrevPhys2Virt = pCur;
4461	return 0;
4462	}
4463
4464
4465	/**
4466	* Perform an integrity check on the PGM component.
4467	*
4468	* @returns VINF_SUCCESS if everything is fine.
4469	* @returns VBox error status after asserting on integrity breach.
4470	* @param pVM The VM handle.
4471	*/
4472	VMMR3DECL(int) PGMR3CheckIntegrity(PVM pVM)
4473	{
4474	AssertReleaseReturn(pVM->pgm.s.offVM, VERR_INTERNAL_ERROR);
4475
4476	/*
4477	* Check the trees.
4478	*/
4479	int cErrors = 0;
4480	const static PGMCHECKINTARGS s_LeftToRight = { true, NULL, NULL, NULL, pVM };
4481	const static PGMCHECKINTARGS s_RightToLeft = { false, NULL, NULL, NULL, pVM };
4482	PGMCHECKINTARGS Args = s_LeftToRight;
4483	cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers, true, pgmR3CheckIntegrityPhysHandlerNode, &Args);
4484	Args = s_RightToLeft;
4485	cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers, false, pgmR3CheckIntegrityPhysHandlerNode, &Args);
4486	Args = s_LeftToRight;
4487	cErrors += RTAvlroGCPtrDoWithAll( &pVM->pgm.s.pTreesR3->VirtHandlers, true, pgmR3CheckIntegrityVirtHandlerNode, &Args);
4488	Args = s_RightToLeft;
4489	cErrors += RTAvlroGCPtrDoWithAll( &pVM->pgm.s.pTreesR3->VirtHandlers, false, pgmR3CheckIntegrityVirtHandlerNode, &Args);
4490	Args = s_LeftToRight;
4491	cErrors += RTAvlroGCPtrDoWithAll( &pVM->pgm.s.pTreesR3->HyperVirtHandlers, true, pgmR3CheckIntegrityVirtHandlerNode, &Args);
4492	Args = s_RightToLeft;
4493	cErrors += RTAvlroGCPtrDoWithAll( &pVM->pgm.s.pTreesR3->HyperVirtHandlers, false, pgmR3CheckIntegrityVirtHandlerNode, &Args);
4494	Args = s_LeftToRight;
4495	cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysToVirtHandlers, true, pgmR3CheckIntegrityPhysToVirtHandlerNode, &Args);
4496	Args = s_RightToLeft;
4497	cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysToVirtHandlers, false, pgmR3CheckIntegrityPhysToVirtHandlerNode, &Args);
4498
4499	return !cErrors ? VINF_SUCCESS : VERR_INTERNAL_ERROR;
4500	}
4501

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/PGM.cpp@ 27184

Download in other formats: