PGM.cpp@ 21134

Last change on this file since 21134 was 20925, checked in by vboxsync, 16 years ago
Reset paging mode back to real mode before resetting the pgm pool. Previous code triggered assertions as the CPU state was already reset and inconsistent with the paging mode.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 230.7 KB

Line
1	/* $Id: PGM.cpp 20925 2009-06-25 11:25:09Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor. (Mixing stuff here, not good?)
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/** @page pg_pgm PGM - The Page Manager and Monitor
24	*
25	* @see grp_pgm,
26	* @ref pg_pgm_pool,
27	* @ref pg_pgm_phys.
28	*
29	*
30	* @section sec_pgm_modes Paging Modes
31	*
32	* There are three memory contexts: Host Context (HC), Guest Context (GC)
33	* and intermediate context. When talking about paging HC can also be refered to
34	* as "host paging", and GC refered to as "shadow paging".
35	*
36	* We define three basic paging modes: 32-bit, PAE and AMD64. The host paging mode
37	* is defined by the host operating system. The mode used in the shadow paging mode
38	* depends on the host paging mode and what the mode the guest is currently in. The
39	* following relation between the two is defined:
40	*
41	* @verbatim
42	Host > 32-bit \| PAE \| AMD64 \|
43	Guest \| \| \| \|
44	==v================================
45	32-bit 32-bit PAE PAE
46	-------\|--------\|--------\|--------\|
47	PAE PAE PAE PAE
48	-------\|--------\|--------\|--------\|
49	AMD64 AMD64 AMD64 AMD64
50	-------\|--------\|--------\|--------\| @endverbatim
51	*
52	* All configuration except those in the diagonal (upper left) are expected to
53	* require special effort from the switcher (i.e. a bit slower).
54	*
55	*
56	*
57	*
58	* @section sec_pgm_shw The Shadow Memory Context
59	*
60	*
61	* [..]
62	*
63	* Because of guest context mappings requires PDPT and PML4 entries to allow
64	* writing on AMD64, the two upper levels will have fixed flags whatever the
65	* guest is thinking of using there. So, when shadowing the PD level we will
66	* calculate the effective flags of PD and all the higher levels. In legacy
67	* PAE mode this only applies to the PWT and PCD bits (the rest are
68	* ignored/reserved/MBZ). We will ignore those bits for the present.
69	*
70	*
71	*
72	* @section sec_pgm_int The Intermediate Memory Context
73	*
74	* The world switch goes thru an intermediate memory context which purpose it is
75	* to provide different mappings of the switcher code. All guest mappings are also
76	* present in this context.
77	*
78	* The switcher code is mapped at the same location as on the host, at an
79	* identity mapped location (physical equals virtual address), and at the
80	* hypervisor location. The identity mapped location is for when the world
81	* switches that involves disabling paging.
82	*
83	* PGM maintain page tables for 32-bit, PAE and AMD64 paging modes. This
84	* simplifies switching guest CPU mode and consistency at the cost of more
85	* code to do the work. All memory use for those page tables is located below
86	* 4GB (this includes page tables for guest context mappings).
87	*
88	*
89	* @subsection subsec_pgm_int_gc Guest Context Mappings
90	*
91	* During assignment and relocation of a guest context mapping the intermediate
92	* memory context is used to verify the new location.
93	*
94	* Guest context mappings are currently restricted to below 4GB, for reasons
95	* of simplicity. This may change when we implement AMD64 support.
96	*
97	*
98	*
99	*
100	* @section sec_pgm_misc Misc
101	*
102	* @subsection subsec_pgm_misc_diff Differences Between Legacy PAE and Long Mode PAE
103	*
104	* The differences between legacy PAE and long mode PAE are:
105	* -# PDPE bits 1, 2, 5 and 6 are defined differently. In leagcy mode they are
106	* all marked down as must-be-zero, while in long mode 1, 2 and 5 have the
107	* usual meanings while 6 is ignored (AMD). This means that upon switching to
108	* legacy PAE mode we'll have to clear these bits and when going to long mode
109	* they must be set. This applies to both intermediate and shadow contexts,
110	* however we don't need to do it for the intermediate one since we're
111	* executing with CR0.WP at that time.
112	* -# CR3 allows a 32-byte aligned address in legacy mode, while in long mode
113	* a page aligned one is required.
114	*
115	*
116	* @section sec_pgm_handlers Access Handlers
117	*
118	* Placeholder.
119	*
120	*
121	* @subsection sec_pgm_handlers_virt Virtual Access Handlers
122	*
123	* Placeholder.
124	*
125	*
126	* @subsection sec_pgm_handlers_virt Virtual Access Handlers
127	*
128	* We currently implement three types of virtual access handlers: ALL, WRITE
129	* and HYPERVISOR (WRITE). See PGMVIRTHANDLERTYPE for some more details.
130	*
131	* The HYPERVISOR access handlers is kept in a separate tree since it doesn't apply
132	* to physical pages (PGMTREES::HyperVirtHandlers) and only needs to be consulted in
133	* a special \#PF case. The ALL and WRITE are in the PGMTREES::VirtHandlers tree, the
134	* rest of this section is going to be about these handlers.
135	*
136	* We'll go thru the life cycle of a handler and try make sense of it all, don't know
137	* how successfull this is gonna be...
138	*
139	* 1. A handler is registered thru the PGMR3HandlerVirtualRegister and
140	* PGMHandlerVirtualRegisterEx APIs. We check for conflicting virtual handlers
141	* and create a new node that is inserted into the AVL tree (range key). Then
142	* a full PGM resync is flagged (clear pool, sync cr3, update virtual bit of PGMPAGE).
143	*
144	* 2. The following PGMSyncCR3/SyncCR3 operation will first make invoke HandlerVirtualUpdate.
145	*
146	* 2a. HandlerVirtualUpdate will will lookup all the pages covered by virtual handlers
147	* via the current guest CR3 and update the physical page -> virtual handler
148	* translation. Needless to say, this doesn't exactly scale very well. If any changes
149	* are detected, it will flag a virtual bit update just like we did on registration.
150	* PGMPHYS pages with changes will have their virtual handler state reset to NONE.
151	*
152	* 2b. The virtual bit update process will iterate all the pages covered by all the
153	* virtual handlers and update the PGMPAGE virtual handler state to the max of all
154	* virtual handlers on that page.
155	*
156	* 2c. Back in SyncCR3 we will now flush the entire shadow page cache to make sure
157	* we don't miss any alias mappings of the monitored pages.
158	*
159	* 2d. SyncCR3 will then proceed with syncing the CR3 table.
160	*
161	* 3. \#PF(np,read) on a page in the range. This will cause it to be synced
162	* read-only and resumed if it's a WRITE handler. If it's an ALL handler we
163	* will call the handlers like in the next step. If the physical mapping has
164	* changed we will - some time in the future - perform a handler callback
165	* (optional) and update the physical -> virtual handler cache.
166	*
167	* 4. \#PF(,write) on a page in the range. This will cause the handler to
168	* be invoked.
169	*
170	* 5. The guest invalidates the page and changes the physical backing or
171	* unmaps it. This should cause the invalidation callback to be invoked
172	* (it might not yet be 100% perfect). Exactly what happens next... is
173	* this where we mess up and end up out of sync for a while?
174	*
175	* 6. The handler is deregistered by the client via PGMHandlerVirtualDeregister.
176	* We will then set all PGMPAGEs in the physical -> virtual handler cache for
177	* this handler to NONE and trigger a full PGM resync (basically the same
178	* as int step 1). Which means 2 is executed again.
179	*
180	*
181	* @subsubsection sub_sec_pgm_handler_virt_todo TODOs
182	*
183	* There is a bunch of things that needs to be done to make the virtual handlers
184	* work 100% correctly and work more efficiently.
185	*
186	* The first bit hasn't been implemented yet because it's going to slow the
187	* whole mess down even more, and besides it seems to be working reliably for
188	* our current uses. OTOH, some of the optimizations might end up more or less
189	* implementing the missing bits, so we'll see.
190	*
191	* On the optimization side, the first thing to do is to try avoid unnecessary
192	* cache flushing. Then try team up with the shadowing code to track changes
193	* in mappings by means of access to them (shadow in), updates to shadows pages,
194	* invlpg, and shadow PT discarding (perhaps).
195	*
196	* Some idea that have popped up for optimization for current and new features:
197	* - bitmap indicating where there are virtual handlers installed.
198	* (4KB => 220 pages, page 212 => covers 32-bit address space 1:1!)
199	* - Further optimize this by min/max (needs min/max avl getters).
200	* - Shadow page table entry bit (if any left)?
201	*
202	*/
203
204
205	/** @page pg_pgm_phys PGM Physical Guest Memory Management
206	*
207	*
208	* Objectives:
209	* - Guest RAM over-commitment using memory ballooning,
210	* zero pages and general page sharing.
211	* - Moving or mirroring a VM onto a different physical machine.
212	*
213	*
214	* @subsection subsec_pgmPhys_Definitions Definitions
215	*
216	* Allocation chunk - A RTR0MemObjAllocPhysNC object and the tracking
217	* machinery assoicated with it.
218	*
219	*
220	*
221	*
222	* @subsection subsec_pgmPhys_AllocPage Allocating a page.
223	*
224	* Initially we map all guest memory to the (per VM) zero page, which
225	* means that none of the read functions will cause pages to be allocated.
226	*
227	* Exception, access bit in page tables that have been shared. This must
228	* be handled, but we must also make sure PGMGst*Modify doesn't make
229	* unnecessary modifications.
230	*
231	* Allocation points:
232	* - PGMPhysSimpleWriteGCPhys and PGMPhysWrite.
233	* - Replacing a zero page mapping at \#PF.
234	* - Replacing a shared page mapping at \#PF.
235	* - ROM registration (currently MMR3RomRegister).
236	* - VM restore (pgmR3Load).
237	*
238	* For the first three it would make sense to keep a few pages handy
239	* until we've reached the max memory commitment for the VM.
240	*
241	* For the ROM registration, we know exactly how many pages we need
242	* and will request these from ring-0. For restore, we will save
243	* the number of non-zero pages in the saved state and allocate
244	* them up front. This would allow the ring-0 component to refuse
245	* the request if the isn't sufficient memory available for VM use.
246	*
247	* Btw. for both ROM and restore allocations we won't be requiring
248	* zeroed pages as they are going to be filled instantly.
249	*
250	*
251	* @subsection subsec_pgmPhys_FreePage Freeing a page
252	*
253	* There are a few points where a page can be freed:
254	* - After being replaced by the zero page.
255	* - After being replaced by a shared page.
256	* - After being ballooned by the guest additions.
257	* - At reset.
258	* - At restore.
259	*
260	* When freeing one or more pages they will be returned to the ring-0
261	* component and replaced by the zero page.
262	*
263	* The reasoning for clearing out all the pages on reset is that it will
264	* return us to the exact same state as on power on, and may thereby help
265	* us reduce the memory load on the system. Further it might have a
266	* (temporary) positive influence on memory fragmentation (@see subsec_pgmPhys_Fragmentation).
267	*
268	* On restore, as mention under the allocation topic, pages should be
269	* freed / allocated depending on how many is actually required by the
270	* new VM state. The simplest approach is to do like on reset, and free
271	* all non-ROM pages and then allocate what we need.
272	*
273	* A measure to prevent some fragmentation, would be to let each allocation
274	* chunk have some affinity towards the VM having allocated the most pages
275	* from it. Also, try make sure to allocate from allocation chunks that
276	* are almost full. Admittedly, both these measures might work counter to
277	* our intentions and its probably not worth putting a lot of effort,
278	* cpu time or memory into this.
279	*
280	*
281	* @subsection subsec_pgmPhys_SharePage Sharing a page
282	*
283	* The basic idea is that there there will be a idle priority kernel
284	* thread walking the non-shared VM pages hashing them and looking for
285	* pages with the same checksum. If such pages are found, it will compare
286	* them byte-by-byte to see if they actually are identical. If found to be
287	* identical it will allocate a shared page, copy the content, check that
288	* the page didn't change while doing this, and finally request both the
289	* VMs to use the shared page instead. If the page is all zeros (special
290	* checksum and byte-by-byte check) it will request the VM that owns it
291	* to replace it with the zero page.
292	*
293	* To make this efficient, we will have to make sure not to try share a page
294	* that will change its contents soon. This part requires the most work.
295	* A simple idea would be to request the VM to write monitor the page for
296	* a while to make sure it isn't modified any time soon. Also, it may
297	* make sense to skip pages that are being write monitored since this
298	* information is readily available to the thread if it works on the
299	* per-VM guest memory structures (presently called PGMRAMRANGE).
300	*
301	*
302	* @subsection subsec_pgmPhys_Fragmentation Fragmentation Concerns and Counter Measures
303	*
304	* The pages are organized in allocation chunks in ring-0, this is a necessity
305	* if we wish to have an OS agnostic approach to this whole thing. (On Linux we
306	* could easily work on a page-by-page basis if we liked. Whether this is possible
307	* or efficient on NT I don't quite know.) Fragmentation within these chunks may
308	* become a problem as part of the idea here is that we wish to return memory to
309	* the host system.
310	*
311	* For instance, starting two VMs at the same time, they will both allocate the
312	* guest memory on-demand and if permitted their page allocations will be
313	* intermixed. Shut down one of the two VMs and it will be difficult to return
314	* any memory to the host system because the page allocation for the two VMs are
315	* mixed up in the same allocation chunks.
316	*
317	* To further complicate matters, when pages are freed because they have been
318	* ballooned or become shared/zero the whole idea is that the page is supposed
319	* to be reused by another VM or returned to the host system. This will cause
320	* allocation chunks to contain pages belonging to different VMs and prevent
321	* returning memory to the host when one of those VM shuts down.
322	*
323	* The only way to really deal with this problem is to move pages. This can
324	* either be done at VM shutdown and or by the idle priority worker thread
325	* that will be responsible for finding sharable/zero pages. The mechanisms
326	* involved for coercing a VM to move a page (or to do it for it) will be
327	* the same as when telling it to share/zero a page.
328	*
329	*
330	* @subsection subsec_pgmPhys_Tracking Tracking Structures And Their Cost
331	*
332	* There's a difficult balance between keeping the per-page tracking structures
333	* (global and guest page) easy to use and keeping them from eating too much
334	* memory. We have limited virtual memory resources available when operating in
335	* 32-bit kernel space (on 64-bit there'll it's quite a different story). The
336	* tracking structures will be attemted designed such that we can deal with up
337	* to 32GB of memory on a 32-bit system and essentially unlimited on 64-bit ones.
338	*
339	*
340	* @subsubsection subsubsec_pgmPhys_Tracking_Kernel Kernel Space
341	*
342	* @see pg_GMM
343	*
344	* @subsubsection subsubsec_pgmPhys_Tracking_PerVM Per-VM
345	*
346	* Fixed info is the physical address of the page (HCPhys) and the page id
347	* (described above). Theoretically we'll need 48(-12) bits for the HCPhys part.
348	* Today we've restricting ourselves to 40(-12) bits because this is the current
349	* restrictions of all AMD64 implementations (I think Barcelona will up this
350	* to 48(-12) bits, not that it really matters) and I needed the bits for
351	* tracking mappings of a page. 48-12 = 36. That leaves 28 bits, which means a
352	* decent range for the page id: 2^(28+12) = 1024TB.
353	*
354	* In additions to these, we'll have to keep maintaining the page flags as we
355	* currently do. Although it wouldn't harm to optimize these quite a bit, like
356	* for instance the ROM shouldn't depend on having a write handler installed
357	* in order for it to become read-only. A RO/RW bit should be considered so
358	* that the page syncing code doesn't have to mess about checking multiple
359	* flag combinations (ROM \|\| RW handler \|\| write monitored) in order to
360	* figure out how to setup a shadow PTE. But this of course, is second
361	* priority at present. Current this requires 12 bits, but could probably
362	* be optimized to ~8.
363	*
364	* Then there's the 24 bits used to track which shadow page tables are
365	* currently mapping a page for the purpose of speeding up physical
366	* access handlers, and thereby the page pool cache. More bit for this
367	* purpose wouldn't hurt IIRC.
368	*
369	* Then there is a new bit in which we need to record what kind of page
370	* this is, shared, zero, normal or write-monitored-normal. This'll
371	* require 2 bits. One bit might be needed for indicating whether a
372	* write monitored page has been written to. And yet another one or
373	* two for tracking migration status. 3-4 bits total then.
374	*
375	* Whatever is left will can be used to record the sharabilitiy of a
376	* page. The page checksum will not be stored in the per-VM table as
377	* the idle thread will not be permitted to do modifications to it.
378	* It will instead have to keep its own working set of potentially
379	* shareable pages and their check sums and stuff.
380	*
381	* For the present we'll keep the current packing of the
382	* PGMRAMRANGE::aHCPhys to keep the changes simple, only of course,
383	* we'll have to change it to a struct with a total of 128-bits at
384	* our disposal.
385	*
386	* The initial layout will be like this:
387	* @verbatim
388	RTHCPHYS HCPhys; The current stuff.
389	63:40 Current shadow PT tracking stuff.
390	39:12 The physical page frame number.
391	11:0 The current flags.
392	uint32_t u28PageId : 28; The page id.
393	uint32_t u2State : 2; The page state { zero, shared, normal, write monitored }.
394	uint32_t fWrittenTo : 1; Whether a write monitored page was written to.
395	uint32_t u1Reserved : 1; Reserved for later.
396	uint32_t u32Reserved; Reserved for later, mostly sharing stats.
397	@endverbatim
398	*
399	* The final layout will be something like this:
400	* @verbatim
401	RTHCPHYS HCPhys; The current stuff.
402	63:48 High page id (12+).
403	47:12 The physical page frame number.
404	11:0 Low page id.
405	uint32_t fReadOnly : 1; Whether it's readonly page (rom or monitored in some way).
406	uint32_t u3Type : 3; The page type {RESERVED, MMIO, MMIO2, ROM, shadowed ROM, RAM}.
407	uint32_t u2PhysMon : 2; Physical access handler type {none, read, write, all}.
408	uint32_t u2VirtMon : 2; Virtual access handler type {none, read, write, all}..
409	uint32_t u2State : 2; The page state { zero, shared, normal, write monitored }.
410	uint32_t fWrittenTo : 1; Whether a write monitored page was written to.
411	uint32_t u20Reserved : 20; Reserved for later, mostly sharing stats.
412	uint32_t u32Tracking; The shadow PT tracking stuff, roughly.
413	@endverbatim
414	*
415	* Cost wise, this means we'll double the cost for guest memory. There isn't anyway
416	* around that I'm afraid. It means that the cost of dealing out 32GB of memory
417	* to one or more VMs is: (32GB >> PAGE_SHIFT) * 16 bytes, or 128MBs. Or another
418	* example, the VM heap cost when assigning 1GB to a VM will be: 4MB.
419	*
420	* A couple of cost examples for the total cost per-VM + kernel.
421	* 32-bit Windows and 32-bit linux:
422	* 1GB guest ram, 256K pages: 4MB + 2MB(+) = 6MB
423	* 4GB guest ram, 1M pages: 16MB + 8MB(+) = 24MB
424	* 32GB guest ram, 8M pages: 128MB + 64MB(+) = 192MB
425	* 64-bit Windows and 64-bit linux:
426	* 1GB guest ram, 256K pages: 4MB + 3MB(+) = 7MB
427	* 4GB guest ram, 1M pages: 16MB + 12MB(+) = 28MB
428	* 32GB guest ram, 8M pages: 128MB + 96MB(+) = 224MB
429	*
430	* UPDATE - 2007-09-27:
431	* Will need a ballooned flag/state too because we cannot
432	* trust the guest 100% and reporting the same page as ballooned more
433	* than once will put the GMM off balance.
434	*
435	*
436	* @subsection subsec_pgmPhys_Serializing Serializing Access
437	*
438	* Initially, we'll try a simple scheme:
439	*
440	* - The per-VM RAM tracking structures (PGMRAMRANGE) is only modified
441	* by the EMT thread of that VM while in the pgm critsect.
442	* - Other threads in the VM process that needs to make reliable use of
443	* the per-VM RAM tracking structures will enter the critsect.
444	* - No process external thread or kernel thread will ever try enter
445	* the pgm critical section, as that just won't work.
446	* - The idle thread (and similar threads) doesn't not need 100% reliable
447	* data when performing it tasks as the EMT thread will be the one to
448	* do the actual changes later anyway. So, as long as it only accesses
449	* the main ram range, it can do so by somehow preventing the VM from
450	* being destroyed while it works on it...
451	*
452	* - The over-commitment management, including the allocating/freeing
453	* chunks, is serialized by a ring-0 mutex lock (a fast one since the
454	* more mundane mutex implementation is broken on Linux).
455	* - A separeate mutex is protecting the set of allocation chunks so
456	* that pages can be shared or/and freed up while some other VM is
457	* allocating more chunks. This mutex can be take from under the other
458	* one, but not the otherway around.
459	*
460	*
461	* @subsection subsec_pgmPhys_Request VM Request interface
462	*
463	* When in ring-0 it will become necessary to send requests to a VM so it can
464	* for instance move a page while defragmenting during VM destroy. The idle
465	* thread will make use of this interface to request VMs to setup shared
466	* pages and to perform write monitoring of pages.
467	*
468	* I would propose an interface similar to the current VMReq interface, similar
469	* in that it doesn't require locking and that the one sending the request may
470	* wait for completion if it wishes to. This shouldn't be very difficult to
471	* realize.
472	*
473	* The requests themselves are also pretty simple. They are basically:
474	* -# Check that some precondition is still true.
475	* -# Do the update.
476	* -# Update all shadow page tables involved with the page.
477	*
478	* The 3rd step is identical to what we're already doing when updating a
479	* physical handler, see pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs.
480	*
481	*
482	*
483	* @section sec_pgmPhys_MappingCaches Mapping Caches
484	*
485	* In order to be able to map in and out memory and to be able to support
486	* guest with more RAM than we've got virtual address space, we'll employing
487	* a mapping cache. There is already a tiny one for GC (see PGMGCDynMapGCPageEx)
488	* and we'll create a similar one for ring-0 unless we decide to setup a dedicate
489	* memory context for the HWACCM execution.
490	*
491	*
492	* @subsection subsec_pgmPhys_MappingCaches_R3 Ring-3
493	*
494	* We've considered implementing the ring-3 mapping cache page based but found
495	* that this was bother some when one had to take into account TLBs+SMP and
496	* portability (missing the necessary APIs on several platforms). There were
497	* also some performance concerns with this approach which hadn't quite been
498	* worked out.
499	*
500	* Instead, we'll be mapping allocation chunks into the VM process. This simplifies
501	* matters greatly quite a bit since we don't need to invent any new ring-0 stuff,
502	* only some minor RTR0MEMOBJ mapping stuff. The main concern here is that mapping
503	* compared to the previous idea is that mapping or unmapping a 1MB chunk is more
504	* costly than a single page, although how much more costly is uncertain. We'll
505	* try address this by using a very big cache, preferably bigger than the actual
506	* VM RAM size if possible. The current VM RAM sizes should give some idea for
507	* 32-bit boxes, while on 64-bit we can probably get away with employing an
508	* unlimited cache.
509	*
510	* The cache have to parts, as already indicated, the ring-3 side and the
511	* ring-0 side.
512	*
513	* The ring-0 will be tied to the page allocator since it will operate on the
514	* memory objects it contains. It will therefore require the first ring-0 mutex
515	* discussed in @ref subsec_pgmPhys_Serializing. We
516	* some double house keeping wrt to who has mapped what I think, since both
517	* VMMR0.r0 and RTR0MemObj will keep track of mapping relataions
518	*
519	* The ring-3 part will be protected by the pgm critsect. For simplicity, we'll
520	* require anyone that desires to do changes to the mapping cache to do that
521	* from within this critsect. Alternatively, we could employ a separate critsect
522	* for serializing changes to the mapping cache as this would reduce potential
523	* contention with other threads accessing mappings unrelated to the changes
524	* that are in process. We can see about this later, contention will show
525	* up in the statistics anyway, so it'll be simple to tell.
526	*
527	* The organization of the ring-3 part will be very much like how the allocation
528	* chunks are organized in ring-0, that is in an AVL tree by chunk id. To avoid
529	* having to walk the tree all the time, we'll have a couple of lookaside entries
530	* like in we do for I/O ports and MMIO in IOM.
531	*
532	* The simplified flow of a PGMPhysRead/Write function:
533	* -# Enter the PGM critsect.
534	* -# Lookup GCPhys in the ram ranges and get the Page ID.
535	* -# Calc the Allocation Chunk ID from the Page ID.
536	* -# Check the lookaside entries and then the AVL tree for the Chunk ID.
537	* If not found in cache:
538	* -# Call ring-0 and request it to be mapped and supply
539	* a chunk to be unmapped if the cache is maxed out already.
540	* -# Insert the new mapping into the AVL tree (id + R3 address).
541	* -# Update the relevant lookaside entry and return the mapping address.
542	* -# Do the read/write according to monitoring flags and everything.
543	* -# Leave the critsect.
544	*
545	*
546	* @section sec_pgmPhys_Fallback Fallback
547	*
548	* Current all the "second tier" hosts will not support the RTR0MemObjAllocPhysNC
549	* API and thus require a fallback.
550	*
551	* So, when RTR0MemObjAllocPhysNC returns VERR_NOT_SUPPORTED the page allocator
552	* will return to the ring-3 caller (and later ring-0) and asking it to seed
553	* the page allocator with some fresh pages (VERR_GMM_SEED_ME). Ring-3 will
554	* then perform an SUPR3PageAlloc(cbChunk >> PAGE_SHIFT) call and make a
555	* "SeededAllocPages" call to ring-0.
556	*
557	* The first time ring-0 sees the VERR_NOT_SUPPORTED failure it will disable
558	* all page sharing (zero page detection will continue). It will also force
559	* all allocations to come from the VM which seeded the page. Both these
560	* measures are taken to make sure that there will never be any need for
561	* mapping anything into ring-3 - everything will be mapped already.
562	*
563	* Whether we'll continue to use the current MM locked memory management
564	* for this I don't quite know (I'd prefer not to and just ditch that all
565	* togther), we'll see what's simplest to do.
566	*
567	*
568	*
569	* @section sec_pgmPhys_Changes Changes
570	*
571	* Breakdown of the changes involved?
572	*/
573
574	/*******************************************************************************
575	* Header Files *
576	*******************************************************************************/
577	#define LOG_GROUP LOG_GROUP_PGM
578	#include <VBox/dbgf.h>
579	#include <VBox/pgm.h>
580	#include <VBox/cpum.h>
581	#include <VBox/iom.h>
582	#include <VBox/sup.h>
583	#include <VBox/mm.h>
584	#include <VBox/em.h>
585	#include <VBox/stam.h>
586	#include <VBox/rem.h>
587	#include <VBox/dbgf.h>
588	#include <VBox/rem.h>
589	#include <VBox/selm.h>
590	#include <VBox/ssm.h>
591	#include "PGMInternal.h"
592	#include <VBox/vm.h>
593	#include <VBox/dbg.h>
594	#include <VBox/hwaccm.h>
595
596	#include <iprt/assert.h>
597	#include <iprt/alloc.h>
598	#include <iprt/asm.h>
599	#include <iprt/thread.h>
600	#include <iprt/string.h>
601	#ifdef DEBUG_bird
602	# include <iprt/env.h>
603	#endif
604	#include <VBox/param.h>
605	#include <VBox/err.h>
606
607
608	/*******************************************************************************
609	* Defined Constants And Macros *
610	*******************************************************************************/
611	/** Saved state data unit version for 2.5.x and later. */
612	#define PGM_SAVED_STATE_VERSION 9
613	/** Saved state data unit version for 2.2.2 and later. */
614	#define PGM_SAVED_STATE_VERSION_2_2_2 8
615	/** Saved state data unit version for 2.2.0. */
616	#define PGM_SAVED_STATE_VERSION_RR_DESC 7
617	/** Saved state data unit version. */
618	#define PGM_SAVED_STATE_VERSION_OLD_PHYS_CODE 6
619
620
621	/*******************************************************************************
622	* Internal Functions *
623	*******************************************************************************/
624	static int pgmR3InitPaging(PVM pVM);
625	static void pgmR3InitStats(PVM pVM);
626	static DECLCALLBACK(void) pgmR3PhysInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
627	static DECLCALLBACK(void) pgmR3InfoMode(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
628	static DECLCALLBACK(void) pgmR3InfoCr3(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
629	static DECLCALLBACK(int) pgmR3RelocatePhysHandler(PAVLROGCPHYSNODECORE pNode, void *pvUser);
630	static DECLCALLBACK(int) pgmR3RelocateVirtHandler(PAVLROGCPTRNODECORE pNode, void *pvUser);
631	static DECLCALLBACK(int) pgmR3RelocateHyperVirtHandler(PAVLROGCPTRNODECORE pNode, void *pvUser);
632	#ifdef VBOX_STRICT
633	static DECLCALLBACK(void) pgmR3ResetNoMorePhysWritesFlag(PVM pVM, VMSTATE enmState, VMSTATE enmOldState, void *pvUser);
634	#endif
635	static DECLCALLBACK(int) pgmR3Save(PVM pVM, PSSMHANDLE pSSM);
636	static DECLCALLBACK(int) pgmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t u32Version);
637	static int pgmR3ModeDataInit(PVM pVM, bool fResolveGCAndR0);
638	static void pgmR3ModeDataSwitch(PVM pVM, PVMCPU pVCpu, PGMMODE enmShw, PGMMODE enmGst);
639	static PGMMODE pgmR3CalcShadowMode(PVM pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode, VMMSWITCHER *penmSwitcher);
640
641	#ifdef VBOX_WITH_DEBUGGER
642	/** @todo Convert the first two commands to 'info' items. */
643	static DECLCALLBACK(int) pgmR3CmdRam(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
644	static DECLCALLBACK(int) pgmR3CmdMap(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
645	static DECLCALLBACK(int) pgmR3CmdError(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
646	static DECLCALLBACK(int) pgmR3CmdSync(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
647	static DECLCALLBACK(int) pgmR3CmdSyncAlways(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
648	# ifdef VBOX_STRICT
649	static DECLCALLBACK(int) pgmR3CmdAssertCR3(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
650	# endif
651	#endif
652
653
654	/*******************************************************************************
655	* Global Variables *
656	*******************************************************************************/
657	#ifdef VBOX_WITH_DEBUGGER
658	/** Argument descriptors for '.pgmerror' and '.pgmerroroff'. */
659	static const DBGCVARDESC g_aPgmErrorArgs[] =
660	{
661	/* cTimesMin, cTimesMax, enmCategory, fFlags, pszName, pszDescription */
662	{ 0, 1, DBGCVAR_CAT_STRING, 0, "where", "Error injection location." },
663	};
664
665	/** Command descriptors. */
666	static const DBGCCMD g_aCmds[] =
667	{
668	/* pszCmd, cArgsMin, cArgsMax, paArgDesc, cArgDescs, pResultDesc, fFlags, pfnHandler pszSyntax, ....pszDescription */
669	{ "pgmram", 0, 0, NULL, 0, NULL, 0, pgmR3CmdRam, "", "Display the ram ranges." },
670	{ "pgmmap", 0, 0, NULL, 0, NULL, 0, pgmR3CmdMap, "", "Display the mapping ranges." },
671	{ "pgmsync", 0, 0, NULL, 0, NULL, 0, pgmR3CmdSync, "", "Sync the CR3 page." },
672	{ "pgmerror", 0, 1, &g_aPgmErrorArgs[0],1, NULL, 0, pgmR3CmdError, "", "Enables inject runtime of errors into parts of PGM." },
673	{ "pgmerroroff", 0, 1, &g_aPgmErrorArgs[0],1, NULL, 0, pgmR3CmdError, "", "Disables inject runtime errors into parts of PGM." },
674	#ifdef VBOX_STRICT
675	{ "pgmassertcr3", 0, 0, NULL, 0, NULL, 0, pgmR3CmdAssertCR3, "", "Check the shadow CR3 mapping." },
676	#endif
677	{ "pgmsyncalways", 0, 0, NULL, 0, NULL, 0, pgmR3CmdSyncAlways, "", "Toggle permanent CR3 syncing." },
678	};
679	#endif
680
681
682
683
684	/*
685	* Shadow - 32-bit mode
686	*/
687	#define PGM_SHW_TYPE PGM_TYPE_32BIT
688	#define PGM_SHW_NAME(name) PGM_SHW_NAME_32BIT(name)
689	#define PGM_SHW_NAME_RC_STR(name) PGM_SHW_NAME_RC_32BIT_STR(name)
690	#define PGM_SHW_NAME_R0_STR(name) PGM_SHW_NAME_R0_32BIT_STR(name)
691	#include "PGMShw.h"
692
693	/* Guest - real mode */
694	#define PGM_GST_TYPE PGM_TYPE_REAL
695	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
696	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_REAL_STR(name)
697	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_REAL_STR(name)
698	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_REAL(name)
699	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_32BIT_REAL_STR(name)
700	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_32BIT_REAL_STR(name)
701	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
702	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
703	#include "PGMBth.h"
704	#include "PGMGstDefs.h"
705	#include "PGMGst.h"
706	#undef BTH_PGMPOOLKIND_PT_FOR_PT
707	#undef BTH_PGMPOOLKIND_ROOT
708	#undef PGM_BTH_NAME
709	#undef PGM_BTH_NAME_RC_STR
710	#undef PGM_BTH_NAME_R0_STR
711	#undef PGM_GST_TYPE
712	#undef PGM_GST_NAME
713	#undef PGM_GST_NAME_RC_STR
714	#undef PGM_GST_NAME_R0_STR
715
716	/* Guest - protected mode */
717	#define PGM_GST_TYPE PGM_TYPE_PROT
718	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
719	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_PROT_STR(name)
720	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_PROT_STR(name)
721	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name)
722	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_32BIT_PROT_STR(name)
723	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_32BIT_PROT_STR(name)
724	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
725	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
726	#include "PGMBth.h"
727	#include "PGMGstDefs.h"
728	#include "PGMGst.h"
729	#undef BTH_PGMPOOLKIND_PT_FOR_PT
730	#undef BTH_PGMPOOLKIND_ROOT
731	#undef PGM_BTH_NAME
732	#undef PGM_BTH_NAME_RC_STR
733	#undef PGM_BTH_NAME_R0_STR
734	#undef PGM_GST_TYPE
735	#undef PGM_GST_NAME
736	#undef PGM_GST_NAME_RC_STR
737	#undef PGM_GST_NAME_R0_STR
738
739	/* Guest - 32-bit mode */
740	#define PGM_GST_TYPE PGM_TYPE_32BIT
741	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
742	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_32BIT_STR(name)
743	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_32BIT_STR(name)
744	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_32BIT(name)
745	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_32BIT_32BIT_STR(name)
746	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_32BIT_32BIT_STR(name)
747	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
748	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
749	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD
750	#include "PGMBth.h"
751	#include "PGMGstDefs.h"
752	#include "PGMGst.h"
753	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
754	#undef BTH_PGMPOOLKIND_PT_FOR_PT
755	#undef BTH_PGMPOOLKIND_ROOT
756	#undef PGM_BTH_NAME
757	#undef PGM_BTH_NAME_RC_STR
758	#undef PGM_BTH_NAME_R0_STR
759	#undef PGM_GST_TYPE
760	#undef PGM_GST_NAME
761	#undef PGM_GST_NAME_RC_STR
762	#undef PGM_GST_NAME_R0_STR
763
764	#undef PGM_SHW_TYPE
765	#undef PGM_SHW_NAME
766	#undef PGM_SHW_NAME_RC_STR
767	#undef PGM_SHW_NAME_R0_STR
768
769
770	/*
771	* Shadow - PAE mode
772	*/
773	#define PGM_SHW_TYPE PGM_TYPE_PAE
774	#define PGM_SHW_NAME(name) PGM_SHW_NAME_PAE(name)
775	#define PGM_SHW_NAME_RC_STR(name) PGM_SHW_NAME_RC_PAE_STR(name)
776	#define PGM_SHW_NAME_R0_STR(name) PGM_SHW_NAME_R0_PAE_STR(name)
777	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
778	#include "PGMShw.h"
779
780	/* Guest - real mode */
781	#define PGM_GST_TYPE PGM_TYPE_REAL
782	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
783	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_REAL_STR(name)
784	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_REAL_STR(name)
785	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
786	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_PAE_REAL_STR(name)
787	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_PAE_REAL_STR(name)
788	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
789	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
790	#include "PGMGstDefs.h"
791	#include "PGMBth.h"
792	#undef BTH_PGMPOOLKIND_PT_FOR_PT
793	#undef BTH_PGMPOOLKIND_ROOT
794	#undef PGM_BTH_NAME
795	#undef PGM_BTH_NAME_RC_STR
796	#undef PGM_BTH_NAME_R0_STR
797	#undef PGM_GST_TYPE
798	#undef PGM_GST_NAME
799	#undef PGM_GST_NAME_RC_STR
800	#undef PGM_GST_NAME_R0_STR
801
802	/* Guest - protected mode */
803	#define PGM_GST_TYPE PGM_TYPE_PROT
804	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
805	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_PROT_STR(name)
806	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_PROT_STR(name)
807	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name)
808	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_PAE_PROT_STR(name)
809	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_PAE_PROT_STR(name)
810	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
811	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
812	#include "PGMGstDefs.h"
813	#include "PGMBth.h"
814	#undef BTH_PGMPOOLKIND_PT_FOR_PT
815	#undef BTH_PGMPOOLKIND_ROOT
816	#undef PGM_BTH_NAME
817	#undef PGM_BTH_NAME_RC_STR
818	#undef PGM_BTH_NAME_R0_STR
819	#undef PGM_GST_TYPE
820	#undef PGM_GST_NAME
821	#undef PGM_GST_NAME_RC_STR
822	#undef PGM_GST_NAME_R0_STR
823
824	/* Guest - 32-bit mode */
825	#define PGM_GST_TYPE PGM_TYPE_32BIT
826	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
827	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_32BIT_STR(name)
828	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_32BIT_STR(name)
829	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_32BIT(name)
830	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_PAE_32BIT_STR(name)
831	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_PAE_32BIT_STR(name)
832	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
833	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
834	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_FOR_32BIT
835	#include "PGMGstDefs.h"
836	#include "PGMBth.h"
837	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
838	#undef BTH_PGMPOOLKIND_PT_FOR_PT
839	#undef BTH_PGMPOOLKIND_ROOT
840	#undef PGM_BTH_NAME
841	#undef PGM_BTH_NAME_RC_STR
842	#undef PGM_BTH_NAME_R0_STR
843	#undef PGM_GST_TYPE
844	#undef PGM_GST_NAME
845	#undef PGM_GST_NAME_RC_STR
846	#undef PGM_GST_NAME_R0_STR
847
848	/* Guest - PAE mode */
849	#define PGM_GST_TYPE PGM_TYPE_PAE
850	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
851	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_PAE_STR(name)
852	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_PAE_STR(name)
853	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PAE(name)
854	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_PAE_PAE_STR(name)
855	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_PAE_PAE_STR(name)
856	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
857	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
858	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT
859	#include "PGMBth.h"
860	#include "PGMGstDefs.h"
861	#include "PGMGst.h"
862	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
863	#undef BTH_PGMPOOLKIND_PT_FOR_PT
864	#undef BTH_PGMPOOLKIND_ROOT
865	#undef PGM_BTH_NAME
866	#undef PGM_BTH_NAME_RC_STR
867	#undef PGM_BTH_NAME_R0_STR
868	#undef PGM_GST_TYPE
869	#undef PGM_GST_NAME
870	#undef PGM_GST_NAME_RC_STR
871	#undef PGM_GST_NAME_R0_STR
872
873	#undef PGM_SHW_TYPE
874	#undef PGM_SHW_NAME
875	#undef PGM_SHW_NAME_RC_STR
876	#undef PGM_SHW_NAME_R0_STR
877
878
879	/*
880	* Shadow - AMD64 mode
881	*/
882	#define PGM_SHW_TYPE PGM_TYPE_AMD64
883	#define PGM_SHW_NAME(name) PGM_SHW_NAME_AMD64(name)
884	#define PGM_SHW_NAME_RC_STR(name) PGM_SHW_NAME_RC_AMD64_STR(name)
885	#define PGM_SHW_NAME_R0_STR(name) PGM_SHW_NAME_R0_AMD64_STR(name)
886	#include "PGMShw.h"
887
888	#ifdef VBOX_WITH_64_BITS_GUESTS
889	/* Guest - AMD64 mode */
890	# define PGM_GST_TYPE PGM_TYPE_AMD64
891	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
892	# define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_AMD64_STR(name)
893	# define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_AMD64_STR(name)
894	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_AMD64(name)
895	# define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_AMD64_AMD64_STR(name)
896	# define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_AMD64_AMD64_STR(name)
897	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
898	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
899	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_64BIT_PML4
900	# include "PGMBth.h"
901	# include "PGMGstDefs.h"
902	# include "PGMGst.h"
903	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
904	# undef BTH_PGMPOOLKIND_PT_FOR_PT
905	# undef BTH_PGMPOOLKIND_ROOT
906	# undef PGM_BTH_NAME
907	# undef PGM_BTH_NAME_RC_STR
908	# undef PGM_BTH_NAME_R0_STR
909	# undef PGM_GST_TYPE
910	# undef PGM_GST_NAME
911	# undef PGM_GST_NAME_RC_STR
912	# undef PGM_GST_NAME_R0_STR
913	#endif /* VBOX_WITH_64_BITS_GUESTS */
914
915	#undef PGM_SHW_TYPE
916	#undef PGM_SHW_NAME
917	#undef PGM_SHW_NAME_RC_STR
918	#undef PGM_SHW_NAME_R0_STR
919
920
921	/*
922	* Shadow - Nested paging mode
923	*/
924	#define PGM_SHW_TYPE PGM_TYPE_NESTED
925	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED(name)
926	#define PGM_SHW_NAME_RC_STR(name) PGM_SHW_NAME_RC_NESTED_STR(name)
927	#define PGM_SHW_NAME_R0_STR(name) PGM_SHW_NAME_R0_NESTED_STR(name)
928	#include "PGMShw.h"
929
930	/* Guest - real mode */
931	#define PGM_GST_TYPE PGM_TYPE_REAL
932	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
933	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_REAL_STR(name)
934	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_REAL_STR(name)
935	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_REAL(name)
936	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_NESTED_REAL_STR(name)
937	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_NESTED_REAL_STR(name)
938	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
939	#include "PGMGstDefs.h"
940	#include "PGMBth.h"
941	#undef BTH_PGMPOOLKIND_PT_FOR_PT
942	#undef PGM_BTH_NAME
943	#undef PGM_BTH_NAME_RC_STR
944	#undef PGM_BTH_NAME_R0_STR
945	#undef PGM_GST_TYPE
946	#undef PGM_GST_NAME
947	#undef PGM_GST_NAME_RC_STR
948	#undef PGM_GST_NAME_R0_STR
949
950	/* Guest - protected mode */
951	#define PGM_GST_TYPE PGM_TYPE_PROT
952	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
953	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_PROT_STR(name)
954	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_PROT_STR(name)
955	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PROT(name)
956	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_NESTED_PROT_STR(name)
957	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_NESTED_PROT_STR(name)
958	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
959	#include "PGMGstDefs.h"
960	#include "PGMBth.h"
961	#undef BTH_PGMPOOLKIND_PT_FOR_PT
962	#undef PGM_BTH_NAME
963	#undef PGM_BTH_NAME_RC_STR
964	#undef PGM_BTH_NAME_R0_STR
965	#undef PGM_GST_TYPE
966	#undef PGM_GST_NAME
967	#undef PGM_GST_NAME_RC_STR
968	#undef PGM_GST_NAME_R0_STR
969
970	/* Guest - 32-bit mode */
971	#define PGM_GST_TYPE PGM_TYPE_32BIT
972	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
973	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_32BIT_STR(name)
974	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_32BIT_STR(name)
975	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT(name)
976	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_NESTED_32BIT_STR(name)
977	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_NESTED_32BIT_STR(name)
978	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
979	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
980	#include "PGMGstDefs.h"
981	#include "PGMBth.h"
982	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
983	#undef BTH_PGMPOOLKIND_PT_FOR_PT
984	#undef PGM_BTH_NAME
985	#undef PGM_BTH_NAME_RC_STR
986	#undef PGM_BTH_NAME_R0_STR
987	#undef PGM_GST_TYPE
988	#undef PGM_GST_NAME
989	#undef PGM_GST_NAME_RC_STR
990	#undef PGM_GST_NAME_R0_STR
991
992	/* Guest - PAE mode */
993	#define PGM_GST_TYPE PGM_TYPE_PAE
994	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
995	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_PAE_STR(name)
996	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_PAE_STR(name)
997	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE(name)
998	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_NESTED_PAE_STR(name)
999	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_NESTED_PAE_STR(name)
1000	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1001	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
1002	#include "PGMGstDefs.h"
1003	#include "PGMBth.h"
1004	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
1005	#undef BTH_PGMPOOLKIND_PT_FOR_PT
1006	#undef PGM_BTH_NAME
1007	#undef PGM_BTH_NAME_RC_STR
1008	#undef PGM_BTH_NAME_R0_STR
1009	#undef PGM_GST_TYPE
1010	#undef PGM_GST_NAME
1011	#undef PGM_GST_NAME_RC_STR
1012	#undef PGM_GST_NAME_R0_STR
1013
1014	#ifdef VBOX_WITH_64_BITS_GUESTS
1015	/* Guest - AMD64 mode */
1016	# define PGM_GST_TYPE PGM_TYPE_AMD64
1017	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
1018	# define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_AMD64_STR(name)
1019	# define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_AMD64_STR(name)
1020	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64(name)
1021	# define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_NESTED_AMD64_STR(name)
1022	# define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_NESTED_AMD64_STR(name)
1023	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1024	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
1025	# include "PGMGstDefs.h"
1026	# include "PGMBth.h"
1027	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
1028	# undef BTH_PGMPOOLKIND_PT_FOR_PT
1029	# undef PGM_BTH_NAME
1030	# undef PGM_BTH_NAME_RC_STR
1031	# undef PGM_BTH_NAME_R0_STR
1032	# undef PGM_GST_TYPE
1033	# undef PGM_GST_NAME
1034	# undef PGM_GST_NAME_RC_STR
1035	# undef PGM_GST_NAME_R0_STR
1036	#endif /* VBOX_WITH_64_BITS_GUESTS */
1037
1038	#undef PGM_SHW_TYPE
1039	#undef PGM_SHW_NAME
1040	#undef PGM_SHW_NAME_RC_STR
1041	#undef PGM_SHW_NAME_R0_STR
1042
1043
1044	/*
1045	* Shadow - EPT
1046	*/
1047	#define PGM_SHW_TYPE PGM_TYPE_EPT
1048	#define PGM_SHW_NAME(name) PGM_SHW_NAME_EPT(name)
1049	#define PGM_SHW_NAME_RC_STR(name) PGM_SHW_NAME_RC_EPT_STR(name)
1050	#define PGM_SHW_NAME_R0_STR(name) PGM_SHW_NAME_R0_EPT_STR(name)
1051	#include "PGMShw.h"
1052
1053	/* Guest - real mode */
1054	#define PGM_GST_TYPE PGM_TYPE_REAL
1055	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
1056	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_REAL_STR(name)
1057	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_REAL_STR(name)
1058	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_REAL(name)
1059	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_EPT_REAL_STR(name)
1060	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_EPT_REAL_STR(name)
1061	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
1062	#include "PGMGstDefs.h"
1063	#include "PGMBth.h"
1064	#undef BTH_PGMPOOLKIND_PT_FOR_PT
1065	#undef PGM_BTH_NAME
1066	#undef PGM_BTH_NAME_RC_STR
1067	#undef PGM_BTH_NAME_R0_STR
1068	#undef PGM_GST_TYPE
1069	#undef PGM_GST_NAME
1070	#undef PGM_GST_NAME_RC_STR
1071	#undef PGM_GST_NAME_R0_STR
1072
1073	/* Guest - protected mode */
1074	#define PGM_GST_TYPE PGM_TYPE_PROT
1075	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
1076	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_PROT_STR(name)
1077	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_PROT_STR(name)
1078	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name)
1079	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_EPT_PROT_STR(name)
1080	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_EPT_PROT_STR(name)
1081	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
1082	#include "PGMGstDefs.h"
1083	#include "PGMBth.h"
1084	#undef BTH_PGMPOOLKIND_PT_FOR_PT
1085	#undef PGM_BTH_NAME
1086	#undef PGM_BTH_NAME_RC_STR
1087	#undef PGM_BTH_NAME_R0_STR
1088	#undef PGM_GST_TYPE
1089	#undef PGM_GST_NAME
1090	#undef PGM_GST_NAME_RC_STR
1091	#undef PGM_GST_NAME_R0_STR
1092
1093	/* Guest - 32-bit mode */
1094	#define PGM_GST_TYPE PGM_TYPE_32BIT
1095	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
1096	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_32BIT_STR(name)
1097	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_32BIT_STR(name)
1098	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_32BIT(name)
1099	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_EPT_32BIT_STR(name)
1100	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_EPT_32BIT_STR(name)
1101	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
1102	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
1103	#include "PGMGstDefs.h"
1104	#include "PGMBth.h"
1105	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
1106	#undef BTH_PGMPOOLKIND_PT_FOR_PT
1107	#undef PGM_BTH_NAME
1108	#undef PGM_BTH_NAME_RC_STR
1109	#undef PGM_BTH_NAME_R0_STR
1110	#undef PGM_GST_TYPE
1111	#undef PGM_GST_NAME
1112	#undef PGM_GST_NAME_RC_STR
1113	#undef PGM_GST_NAME_R0_STR
1114
1115	/* Guest - PAE mode */
1116	#define PGM_GST_TYPE PGM_TYPE_PAE
1117	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
1118	#define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_PAE_STR(name)
1119	#define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_PAE_STR(name)
1120	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PAE(name)
1121	#define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_EPT_PAE_STR(name)
1122	#define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_EPT_PAE_STR(name)
1123	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1124	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
1125	#include "PGMGstDefs.h"
1126	#include "PGMBth.h"
1127	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
1128	#undef BTH_PGMPOOLKIND_PT_FOR_PT
1129	#undef PGM_BTH_NAME
1130	#undef PGM_BTH_NAME_RC_STR
1131	#undef PGM_BTH_NAME_R0_STR
1132	#undef PGM_GST_TYPE
1133	#undef PGM_GST_NAME
1134	#undef PGM_GST_NAME_RC_STR
1135	#undef PGM_GST_NAME_R0_STR
1136
1137	#ifdef VBOX_WITH_64_BITS_GUESTS
1138	/* Guest - AMD64 mode */
1139	# define PGM_GST_TYPE PGM_TYPE_AMD64
1140	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
1141	# define PGM_GST_NAME_RC_STR(name) PGM_GST_NAME_RC_AMD64_STR(name)
1142	# define PGM_GST_NAME_R0_STR(name) PGM_GST_NAME_R0_AMD64_STR(name)
1143	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_AMD64(name)
1144	# define PGM_BTH_NAME_RC_STR(name) PGM_BTH_NAME_RC_EPT_AMD64_STR(name)
1145	# define PGM_BTH_NAME_R0_STR(name) PGM_BTH_NAME_R0_EPT_AMD64_STR(name)
1146	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1147	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
1148	# include "PGMGstDefs.h"
1149	# include "PGMBth.h"
1150	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
1151	# undef BTH_PGMPOOLKIND_PT_FOR_PT
1152	# undef PGM_BTH_NAME
1153	# undef PGM_BTH_NAME_RC_STR
1154	# undef PGM_BTH_NAME_R0_STR
1155	# undef PGM_GST_TYPE
1156	# undef PGM_GST_NAME
1157	# undef PGM_GST_NAME_RC_STR
1158	# undef PGM_GST_NAME_R0_STR
1159	#endif /* VBOX_WITH_64_BITS_GUESTS */
1160
1161	#undef PGM_SHW_TYPE
1162	#undef PGM_SHW_NAME
1163	#undef PGM_SHW_NAME_RC_STR
1164	#undef PGM_SHW_NAME_R0_STR
1165
1166
1167
1168	/**
1169	* Initiates the paging of VM.
1170	*
1171	* @returns VBox status code.
1172	* @param pVM Pointer to VM structure.
1173	*/
1174	VMMR3DECL(int) PGMR3Init(PVM pVM)
1175	{
1176	LogFlow(("PGMR3Init:\n"));
1177	PCFGMNODE pCfgPGM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/PGM");
1178	int rc;
1179
1180	/*
1181	* Assert alignment and sizes.
1182	*/
1183	AssertCompile(sizeof(pVM->pgm.s) <= sizeof(pVM->pgm.padding));
1184	AssertCompileMemberAlignment(PGM, CritSect, sizeof(uintptr_t));
1185
1186	/*
1187	* Init the structure.
1188	*/
1189	pVM->pgm.s.offVM = RT_OFFSETOF(VM, pgm.s);
1190	pVM->pgm.s.offVCpuPGM = RT_OFFSETOF(VMCPU, pgm.s);
1191
1192	/* Init the per-CPU part. */
1193	for (unsigned i=0;i<pVM->cCPUs;i++)
1194	{
1195	PVMCPU pVCpu = &pVM->aCpus[i];
1196	PPGMCPU pPGM = &pVCpu->pgm.s;
1197
1198	pPGM->offVM = (uintptr_t)&pVCpu->pgm.s - (uintptr_t)pVM;
1199	pPGM->offVCpu = RT_OFFSETOF(VMCPU, pgm.s);
1200	pPGM->offPGM = (uintptr_t)&pVCpu->pgm.s - (uintptr_t)&pVM->pgm.s;
1201
1202	pPGM->enmShadowMode = PGMMODE_INVALID;
1203	pPGM->enmGuestMode = PGMMODE_INVALID;
1204
1205	pPGM->GCPhysCR3 = NIL_RTGCPHYS;
1206
1207	pPGM->pGstPaePdptR3 = NULL;
1208	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
1209	pPGM->pGstPaePdptR0 = NIL_RTR0PTR;
1210	#endif
1211	pPGM->pGstPaePdptRC = NIL_RTRCPTR;
1212	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.apGstPaePDsR3); i++)
1213	{
1214	pPGM->apGstPaePDsR3[i] = NULL;
1215	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
1216	pPGM->apGstPaePDsR0[i] = NIL_RTR0PTR;
1217	#endif
1218	pPGM->apGstPaePDsRC[i] = NIL_RTRCPTR;
1219	pPGM->aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
1220	pPGM->aGCPhysGstPaePDsMonitored[i] = NIL_RTGCPHYS;
1221	}
1222
1223	pPGM->fA20Enabled = true;
1224	}
1225
1226	pVM->pgm.s.enmHostMode = SUPPAGINGMODE_INVALID;
1227	pVM->pgm.s.GCPhys4MBPSEMask = RT_BIT_64(32) - 1; /* default; checked later */
1228	pVM->pgm.s.GCPtrPrevRamRangeMapping = MM_HYPER_AREA_ADDRESS;
1229
1230	rc = CFGMR3QueryBoolDef(CFGMR3GetRoot(pVM), "RamPreAlloc", &pVM->pgm.s.fRamPreAlloc,
1231	#ifdef VBOX_WITH_PREALLOC_RAM_BY_DEFAULT
1232	true
1233	#else
1234	false
1235	#endif
1236	);
1237	AssertLogRelRCReturn(rc, rc);
1238
1239	#if HC_ARCH_BITS == 64 \|\| 1 /** @todo 4GB/32-bit: remove \|\| 1 later and adjust the limit. */
1240	rc = CFGMR3QueryU32Def(pCfgPGM, "MaxRing3Chunks", &pVM->pgm.s.ChunkR3Map.cMax, UINT32_MAX);
1241	#else
1242	rc = CFGMR3QueryU32Def(pCfgPGM, "MaxRing3Chunks", &pVM->pgm.s.ChunkR3Map.cMax, _1G / GMM_CHUNK_SIZE);
1243	#endif
1244	AssertLogRelRCReturn(rc, rc);
1245	for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
1246	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
1247
1248	/*
1249	* Get the configured RAM size - to estimate saved state size.
1250	*/
1251	uint64_t cbRam;
1252	rc = CFGMR3QueryU64(CFGMR3GetRoot(pVM), "RamSize", &cbRam);
1253	if (rc == VERR_CFGM_VALUE_NOT_FOUND)
1254	cbRam = 0;
1255	else if (RT_SUCCESS(rc))
1256	{
1257	if (cbRam < PAGE_SIZE)
1258	cbRam = 0;
1259	cbRam = RT_ALIGN_64(cbRam, PAGE_SIZE);
1260	}
1261	else
1262	{
1263	AssertMsgFailed(("Configuration error: Failed to query integer \"RamSize\", rc=%Rrc.\n", rc));
1264	return rc;
1265	}
1266
1267	/*
1268	* Register callbacks, string formatters and the saved state data unit.
1269	*/
1270	#ifdef VBOX_STRICT
1271	VMR3AtStateRegister(pVM, pgmR3ResetNoMorePhysWritesFlag, NULL);
1272	#endif
1273	PGMRegisterStringFormatTypes();
1274
1275	rc = SSMR3RegisterInternal(pVM, "pgm", 1, PGM_SAVED_STATE_VERSION, (size_t)cbRam + sizeof(PGM),
1276	NULL, pgmR3Save, NULL,
1277	NULL, pgmR3Load, NULL);
1278	if (RT_FAILURE(rc))
1279	return rc;
1280
1281	/*
1282	* Initialize the PGM critical section and flush the phys TLBs
1283	*/
1284	rc = PDMR3CritSectInit(pVM, &pVM->pgm.s.CritSect, "PGM");
1285	AssertRCReturn(rc, rc);
1286
1287	PGMR3PhysChunkInvalidateTLB(pVM);
1288	PGMPhysInvalidatePageR3MapTLB(pVM);
1289	PGMPhysInvalidatePageR0MapTLB(pVM);
1290	PGMPhysInvalidatePageGCMapTLB(pVM);
1291
1292	/*
1293	* For the time being we sport a full set of handy pages in addition to the base
1294	* memory to simplify things.
1295	*/
1296	rc = MMR3ReserveHandyPages(pVM, RT_ELEMENTS(pVM->pgm.s.aHandyPages)); /** @todo this should be changed to PGM_HANDY_PAGES_MIN but this needs proper testing... */
1297	AssertRCReturn(rc, rc);
1298
1299	/*
1300	* Trees
1301	*/
1302	rc = MMHyperAlloc(pVM, sizeof(PGMTREES), 0, MM_TAG_PGM, (void **)&pVM->pgm.s.pTreesR3);
1303	if (RT_SUCCESS(rc))
1304	{
1305	pVM->pgm.s.pTreesR0 = MMHyperR3ToR0(pVM, pVM->pgm.s.pTreesR3);
1306	pVM->pgm.s.pTreesRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pTreesR3);
1307
1308	/*
1309	* Alocate the zero page.
1310	*/
1311	rc = MMHyperAlloc(pVM, PAGE_SIZE, PAGE_SIZE, MM_TAG_PGM, &pVM->pgm.s.pvZeroPgR3);
1312	}
1313	if (RT_SUCCESS(rc))
1314	{
1315	pVM->pgm.s.pvZeroPgRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pvZeroPgR3);
1316	pVM->pgm.s.pvZeroPgR0 = MMHyperR3ToR0(pVM, pVM->pgm.s.pvZeroPgR3);
1317	pVM->pgm.s.HCPhysZeroPg = MMR3HyperHCVirt2HCPhys(pVM, pVM->pgm.s.pvZeroPgR3);
1318	AssertRelease(pVM->pgm.s.HCPhysZeroPg != NIL_RTHCPHYS);
1319
1320	/*
1321	* Init the paging.
1322	*/
1323	rc = pgmR3InitPaging(pVM);
1324	}
1325	if (RT_SUCCESS(rc))
1326	{
1327	/*
1328	* Init the page pool.
1329	*/
1330	rc = pgmR3PoolInit(pVM);
1331	}
1332	if (RT_SUCCESS(rc))
1333	{
1334	for (unsigned i=0;i<pVM->cCPUs;i++)
1335	{
1336	PVMCPU pVCpu = &pVM->aCpus[i];
1337
1338	rc = PGMR3ChangeMode(pVM, pVCpu, PGMMODE_REAL);
1339	if (RT_FAILURE(rc))
1340	break;
1341	}
1342	}
1343
1344	if (RT_SUCCESS(rc))
1345	{
1346	/*
1347	* Info & statistics
1348	*/
1349	DBGFR3InfoRegisterInternal(pVM, "mode",
1350	"Shows the current paging mode. "
1351	"Recognizes 'all', 'guest', 'shadow' and 'host' as arguments, defaulting to 'all' if nothing's given.",
1352	pgmR3InfoMode);
1353	DBGFR3InfoRegisterInternal(pVM, "pgmcr3",
1354	"Dumps all the entries in the top level paging table. No arguments.",
1355	pgmR3InfoCr3);
1356	DBGFR3InfoRegisterInternal(pVM, "phys",
1357	"Dumps all the physical address ranges. No arguments.",
1358	pgmR3PhysInfo);
1359	DBGFR3InfoRegisterInternal(pVM, "handlers",
1360	"Dumps physical, virtual and hyper virtual handlers. "
1361	"Pass 'phys', 'virt', 'hyper' as argument if only one kind is wanted."
1362	"Add 'nost' if the statistics are unwanted, use together with 'all' or explicit selection.",
1363	pgmR3InfoHandlers);
1364	DBGFR3InfoRegisterInternal(pVM, "mappings",
1365	"Dumps guest mappings.",
1366	pgmR3MapInfo);
1367
1368	pgmR3InitStats(pVM);
1369
1370	#ifdef VBOX_WITH_DEBUGGER
1371	/*
1372	* Debugger commands.
1373	*/
1374	static bool s_fRegisteredCmds = false;
1375	if (!s_fRegisteredCmds)
1376	{
1377	int rc = DBGCRegisterCommands(&g_aCmds[0], RT_ELEMENTS(g_aCmds));
1378	if (RT_SUCCESS(rc))
1379	s_fRegisteredCmds = true;
1380	}
1381	#endif
1382	return VINF_SUCCESS;
1383	}
1384
1385	/* Almost no cleanup necessary, MM frees all memory. */
1386	PDMR3CritSectDelete(&pVM->pgm.s.CritSect);
1387
1388	return rc;
1389	}
1390
1391
1392	/**
1393	* Initializes the per-VCPU PGM.
1394	*
1395	* @returns VBox status code.
1396	* @param pVM The VM to operate on.
1397	*/
1398	VMMR3DECL(int) PGMR3InitCPU(PVM pVM)
1399	{
1400	LogFlow(("PGMR3InitCPU\n"));
1401	return VINF_SUCCESS;
1402	}
1403
1404
1405	/**
1406	* Init paging.
1407	*
1408	* Since we need to check what mode the host is operating in before we can choose
1409	* the right paging functions for the host we have to delay this until R0 has
1410	* been initialized.
1411	*
1412	* @returns VBox status code.
1413	* @param pVM VM handle.
1414	*/
1415	static int pgmR3InitPaging(PVM pVM)
1416	{
1417	/*
1418	* Force a recalculation of modes and switcher so everyone gets notified.
1419	*/
1420	for (unsigned i=0;i<pVM->cCPUs;i++)
1421	{
1422	PVMCPU pVCpu = &pVM->aCpus[i];
1423
1424	pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID;
1425	pVCpu->pgm.s.enmGuestMode = PGMMODE_INVALID;
1426	}
1427
1428	pVM->pgm.s.enmHostMode = SUPPAGINGMODE_INVALID;
1429
1430	/*
1431	* Allocate static mapping space for whatever the cr3 register
1432	* points to and in the case of PAE mode to the 4 PDs.
1433	*/
1434	int rc = MMR3HyperReserve(pVM, PAGE_SIZE * 5, "CR3 mapping", &pVM->pgm.s.GCPtrCR3Mapping);
1435	if (RT_FAILURE(rc))
1436	{
1437	AssertMsgFailed(("Failed to reserve two pages for cr mapping in HMA, rc=%Rrc\n", rc));
1438	return rc;
1439	}
1440	MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
1441
1442	/*
1443	* Allocate pages for the three possible intermediate contexts
1444	* (AMD64, PAE and plain 32-Bit). We maintain all three contexts
1445	* for the sake of simplicity. The AMD64 uses the PAE for the
1446	* lower levels, making the total number of pages 11 (3 + 7 + 1).
1447	*
1448	* We assume that two page tables will be enought for the core code
1449	* mappings (HC virtual and identity).
1450	*/
1451	pVM->pgm.s.pInterPD = (PX86PD)MMR3PageAllocLow(pVM); AssertReturn(pVM->pgm.s.pInterPD, VERR_NO_PAGE_MEMORY);
1452	pVM->pgm.s.apInterPTs[0] = (PX86PT)MMR3PageAllocLow(pVM); AssertReturn(pVM->pgm.s.apInterPTs[0], VERR_NO_PAGE_MEMORY);
1453	pVM->pgm.s.apInterPTs[1] = (PX86PT)MMR3PageAllocLow(pVM); AssertReturn(pVM->pgm.s.apInterPTs[1], VERR_NO_PAGE_MEMORY);
1454	pVM->pgm.s.apInterPaePTs[0] = (PX86PTPAE)MMR3PageAlloc(pVM); AssertReturn(pVM->pgm.s.apInterPaePTs[0], VERR_NO_PAGE_MEMORY);
1455	pVM->pgm.s.apInterPaePTs[1] = (PX86PTPAE)MMR3PageAlloc(pVM); AssertReturn(pVM->pgm.s.apInterPaePTs[1], VERR_NO_PAGE_MEMORY);
1456	pVM->pgm.s.apInterPaePDs[0] = (PX86PDPAE)MMR3PageAlloc(pVM); AssertReturn(pVM->pgm.s.apInterPaePDs[0], VERR_NO_PAGE_MEMORY);
1457	pVM->pgm.s.apInterPaePDs[1] = (PX86PDPAE)MMR3PageAlloc(pVM); AssertReturn(pVM->pgm.s.apInterPaePDs[1], VERR_NO_PAGE_MEMORY);
1458	pVM->pgm.s.apInterPaePDs[2] = (PX86PDPAE)MMR3PageAlloc(pVM); AssertReturn(pVM->pgm.s.apInterPaePDs[2], VERR_NO_PAGE_MEMORY);
1459	pVM->pgm.s.apInterPaePDs[3] = (PX86PDPAE)MMR3PageAlloc(pVM); AssertReturn(pVM->pgm.s.apInterPaePDs[3], VERR_NO_PAGE_MEMORY);
1460	pVM->pgm.s.pInterPaePDPT = (PX86PDPT)MMR3PageAllocLow(pVM); AssertReturn(pVM->pgm.s.pInterPaePDPT, VERR_NO_PAGE_MEMORY);
1461	pVM->pgm.s.pInterPaePDPT64 = (PX86PDPT)MMR3PageAllocLow(pVM); AssertReturn(pVM->pgm.s.pInterPaePDPT64, VERR_NO_PAGE_MEMORY);
1462	pVM->pgm.s.pInterPaePML4 = (PX86PML4)MMR3PageAllocLow(pVM); AssertReturn(pVM->pgm.s.pInterPaePML4, VERR_NO_PAGE_MEMORY);
1463
1464	pVM->pgm.s.HCPhysInterPD = MMPage2Phys(pVM, pVM->pgm.s.pInterPD);
1465	AssertRelease(pVM->pgm.s.HCPhysInterPD != NIL_RTHCPHYS && !(pVM->pgm.s.HCPhysInterPD & PAGE_OFFSET_MASK));
1466	pVM->pgm.s.HCPhysInterPaePDPT = MMPage2Phys(pVM, pVM->pgm.s.pInterPaePDPT);
1467	AssertRelease(pVM->pgm.s.HCPhysInterPaePDPT != NIL_RTHCPHYS && !(pVM->pgm.s.HCPhysInterPaePDPT & PAGE_OFFSET_MASK));
1468	pVM->pgm.s.HCPhysInterPaePML4 = MMPage2Phys(pVM, pVM->pgm.s.pInterPaePML4);
1469	AssertRelease(pVM->pgm.s.HCPhysInterPaePML4 != NIL_RTHCPHYS && !(pVM->pgm.s.HCPhysInterPaePML4 & PAGE_OFFSET_MASK) && pVM->pgm.s.HCPhysInterPaePML4 < 0xffffffff);
1470
1471	/*
1472	* Initialize the pages, setting up the PML4 and PDPT for repetitive 4GB action.
1473	*/
1474	ASMMemZeroPage(pVM->pgm.s.pInterPD);
1475	ASMMemZeroPage(pVM->pgm.s.apInterPTs[0]);
1476	ASMMemZeroPage(pVM->pgm.s.apInterPTs[1]);
1477
1478	ASMMemZeroPage(pVM->pgm.s.apInterPaePTs[0]);
1479	ASMMemZeroPage(pVM->pgm.s.apInterPaePTs[1]);
1480
1481	ASMMemZeroPage(pVM->pgm.s.pInterPaePDPT);
1482	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.apInterPaePDs); i++)
1483	{
1484	ASMMemZeroPage(pVM->pgm.s.apInterPaePDs[i]);
1485	pVM->pgm.s.pInterPaePDPT->a[i].u = X86_PDPE_P \| PGM_PLXFLAGS_PERMANENT
1486	\| MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[i]);
1487	}
1488
1489	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.pInterPaePDPT64->a); i++)
1490	{
1491	const unsigned iPD = i % RT_ELEMENTS(pVM->pgm.s.apInterPaePDs);
1492	pVM->pgm.s.pInterPaePDPT64->a[i].u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A \| PGM_PLXFLAGS_PERMANENT
1493	\| MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[iPD]);
1494	}
1495
1496	RTHCPHYS HCPhysInterPaePDPT64 = MMPage2Phys(pVM, pVM->pgm.s.pInterPaePDPT64);
1497	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.pInterPaePML4->a); i++)
1498	pVM->pgm.s.pInterPaePML4->a[i].u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A \| PGM_PLXFLAGS_PERMANENT
1499	\| HCPhysInterPaePDPT64;
1500
1501	/*
1502	* Initialize paging workers and mode from current host mode
1503	* and the guest running in real mode.
1504	*/
1505	pVM->pgm.s.enmHostMode = SUPR3GetPagingMode();
1506	switch (pVM->pgm.s.enmHostMode)
1507	{
1508	case SUPPAGINGMODE_32_BIT:
1509	case SUPPAGINGMODE_32_BIT_GLOBAL:
1510	case SUPPAGINGMODE_PAE:
1511	case SUPPAGINGMODE_PAE_GLOBAL:
1512	case SUPPAGINGMODE_PAE_NX:
1513	case SUPPAGINGMODE_PAE_GLOBAL_NX:
1514	break;
1515
1516	case SUPPAGINGMODE_AMD64:
1517	case SUPPAGINGMODE_AMD64_GLOBAL:
1518	case SUPPAGINGMODE_AMD64_NX:
1519	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
1520	#ifndef VBOX_WITH_HYBRID_32BIT_KERNEL
1521	if (ARCH_BITS != 64)
1522	{
1523	AssertMsgFailed(("Host mode %d (64-bit) is not supported by non-64bit builds\n", pVM->pgm.s.enmHostMode));
1524	LogRel(("Host mode %d (64-bit) is not supported by non-64bit builds\n", pVM->pgm.s.enmHostMode));
1525	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
1526	}
1527	#endif
1528	break;
1529	default:
1530	AssertMsgFailed(("Host mode %d is not supported\n", pVM->pgm.s.enmHostMode));
1531	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
1532	}
1533	rc = pgmR3ModeDataInit(pVM, false /* don't resolve GC and R0 syms yet */);
1534	if (RT_SUCCESS(rc))
1535	{
1536	LogFlow(("pgmR3InitPaging: returns successfully\n"));
1537	#if HC_ARCH_BITS == 64
1538	LogRel(("Debug: HCPhysInterPD=%RHp HCPhysInterPaePDPT=%RHp HCPhysInterPaePML4=%RHp\n",
1539	pVM->pgm.s.HCPhysInterPD, pVM->pgm.s.HCPhysInterPaePDPT, pVM->pgm.s.HCPhysInterPaePML4));
1540	LogRel(("Debug: apInterPTs={%RHp,%RHp} apInterPaePTs={%RHp,%RHp} apInterPaePDs={%RHp,%RHp,%RHp,%RHp} pInterPaePDPT64=%RHp\n",
1541	MMPage2Phys(pVM, pVM->pgm.s.apInterPTs[0]), MMPage2Phys(pVM, pVM->pgm.s.apInterPTs[1]),
1542	MMPage2Phys(pVM, pVM->pgm.s.apInterPaePTs[0]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePTs[1]),
1543	MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[0]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[1]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[2]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[3]),
1544	MMPage2Phys(pVM, pVM->pgm.s.pInterPaePDPT64)));
1545	#endif
1546
1547	return VINF_SUCCESS;
1548	}
1549
1550	LogFlow(("pgmR3InitPaging: returns %Rrc\n", rc));
1551	return rc;
1552	}
1553
1554
1555	/**
1556	* Init statistics
1557	*/
1558	static void pgmR3InitStats(PVM pVM)
1559	{
1560	PPGM pPGM = &pVM->pgm.s;
1561	int rc;
1562
1563	/* Common - misc variables */
1564	STAM_REL_REG(pVM, &pPGM->cAllPages, STAMTYPE_U32, "/PGM/Page/cAllPages", STAMUNIT_OCCURENCES, "The total number of pages.");
1565	STAM_REL_REG(pVM, &pPGM->cPrivatePages, STAMTYPE_U32, "/PGM/Page/cPrivatePages", STAMUNIT_OCCURENCES, "The number of private pages.");
1566	STAM_REL_REG(pVM, &pPGM->cSharedPages, STAMTYPE_U32, "/PGM/Page/cSharedPages", STAMUNIT_OCCURENCES, "The number of shared pages.");
1567	STAM_REL_REG(pVM, &pPGM->cZeroPages, STAMTYPE_U32, "/PGM/Page/cZeroPages", STAMUNIT_OCCURENCES, "The number of zero backed pages.");
1568	STAM_REL_REG(pVM, &pPGM->cHandyPages, STAMTYPE_U32, "/PGM/Page/cHandyPages", STAMUNIT_OCCURENCES, "The number of handy pages (not included in cAllPages).");
1569	STAM_REL_REG(pVM, &pPGM->cRelocations, STAMTYPE_COUNTER, "/PGM/cRelocations", STAMUNIT_OCCURENCES, "Number of hypervisor relocations.");
1570	STAM_REL_REG(pVM, &pPGM->ChunkR3Map.c, STAMTYPE_U32, "/PGM/ChunkR3Map/c", STAMUNIT_OCCURENCES, "Number of mapped chunks.");
1571	STAM_REL_REG(pVM, &pPGM->ChunkR3Map.cMax, STAMTYPE_U32, "/PGM/ChunkR3Map/cMax", STAMUNIT_OCCURENCES, "Maximum number of mapped chunks.");
1572
1573	#ifdef VBOX_WITH_STATISTICS
1574
1575	# define PGM_REG_COUNTER(a, b, c) \
1576	rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, c, b); \
1577	AssertRC(rc);
1578
1579	# define PGM_REG_PROFILE(a, b, c) \
1580	rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b); \
1581	AssertRC(rc);
1582
1583	PGM_REG_COUNTER(&pPGM->StatR3DetectedConflicts, "/PGM/R3/DetectedConflicts", "The number of times PGMR3CheckMappingConflicts() detected a conflict.");
1584	PGM_REG_PROFILE(&pPGM->StatR3ResolveConflict, "/PGM/R3/ResolveConflict", "pgmR3SyncPTResolveConflict() profiling (includes the entire relocation).");
1585
1586	PGM_REG_COUNTER(&pPGM->StatRZChunkR3MapTlbHits, "/PGM/ChunkR3Map/TlbHitsRZ", "TLB hits.");
1587	PGM_REG_COUNTER(&pPGM->StatRZChunkR3MapTlbMisses, "/PGM/ChunkR3Map/TlbMissesRZ", "TLB misses.");
1588	PGM_REG_COUNTER(&pPGM->StatRZPageMapTlbHits, "/PGM/RZ/Page/MapTlbHits", "TLB hits.");
1589	PGM_REG_COUNTER(&pPGM->StatRZPageMapTlbMisses, "/PGM/RZ/Page/MapTlbMisses", "TLB misses.");
1590	PGM_REG_COUNTER(&pPGM->StatR3ChunkR3MapTlbHits, "/PGM/ChunkR3Map/TlbHitsR3", "TLB hits.");
1591	PGM_REG_COUNTER(&pPGM->StatR3ChunkR3MapTlbMisses, "/PGM/ChunkR3Map/TlbMissesR3", "TLB misses.");
1592	PGM_REG_COUNTER(&pPGM->StatR3PageMapTlbHits, "/PGM/R3/Page/MapTlbHits", "TLB hits.");
1593	PGM_REG_COUNTER(&pPGM->StatR3PageMapTlbMisses, "/PGM/R3/Page/MapTlbMisses", "TLB misses.");
1594
1595	PGM_REG_PROFILE(&pPGM->StatRZSyncCR3HandlerVirtualUpdate, "/PGM/RZ/SyncCR3/Handlers/VirtualUpdate", "Profiling of the virtual handler updates.");
1596	PGM_REG_PROFILE(&pPGM->StatRZSyncCR3HandlerVirtualReset, "/PGM/RZ/SyncCR3/Handlers/VirtualReset", "Profiling of the virtual handler resets.");
1597	PGM_REG_PROFILE(&pPGM->StatR3SyncCR3HandlerVirtualUpdate, "/PGM/R3/SyncCR3/Handlers/VirtualUpdate", "Profiling of the virtual handler updates.");
1598	PGM_REG_PROFILE(&pPGM->StatR3SyncCR3HandlerVirtualReset, "/PGM/R3/SyncCR3/Handlers/VirtualReset", "Profiling of the virtual handler resets.");
1599
1600	PGM_REG_COUNTER(&pPGM->StatRZPhysHandlerReset, "/PGM/RZ/PhysHandlerReset", "The number of times PGMHandlerPhysicalReset is called.");
1601	PGM_REG_COUNTER(&pPGM->StatR3PhysHandlerReset, "/PGM/R3/PhysHandlerReset", "The number of times PGMHandlerPhysicalReset is called.");
1602	PGM_REG_PROFILE(&pPGM->StatRZVirtHandlerSearchByPhys, "/PGM/RZ/VirtHandlerSearchByPhys", "Profiling of pgmHandlerVirtualFindByPhysAddr.");
1603	PGM_REG_PROFILE(&pPGM->StatR3VirtHandlerSearchByPhys, "/PGM/R3/VirtHandlerSearchByPhys", "Profiling of pgmHandlerVirtualFindByPhysAddr.");
1604
1605	PGM_REG_COUNTER(&pPGM->StatRZPageReplaceShared, "/PGM/RZ/Page/ReplacedShared", "Times a shared page was replaced.");
1606	PGM_REG_COUNTER(&pPGM->StatRZPageReplaceZero, "/PGM/RZ/Page/ReplacedZero", "Times the zero page was replaced.");
1607	/// @todo PGM_REG_COUNTER(&pPGM->StatRZPageHandyAllocs, "/PGM/RZ/Page/HandyAllocs", "Number of times we've allocated more handy pages.");
1608	PGM_REG_COUNTER(&pPGM->StatR3PageReplaceShared, "/PGM/R3/Page/ReplacedShared", "Times a shared page was replaced.");
1609	PGM_REG_COUNTER(&pPGM->StatR3PageReplaceZero, "/PGM/R3/Page/ReplacedZero", "Times the zero page was replaced.");
1610	/// @todo PGM_REG_COUNTER(&pPGM->StatR3PageHandyAllocs, "/PGM/R3/Page/HandyAllocs", "Number of times we've allocated more handy pages.");
1611
1612	/* GC only: */
1613	PGM_REG_COUNTER(&pPGM->StatRCDynMapCacheHits, "/PGM/RC/DynMapCache/Hits" , "Number of dynamic page mapping cache hits.");
1614	PGM_REG_COUNTER(&pPGM->StatRCDynMapCacheMisses, "/PGM/RC/DynMapCache/Misses" , "Number of dynamic page mapping cache misses.");
1615	PGM_REG_COUNTER(&pPGM->StatRCInvlPgConflict, "/PGM/RC/InvlPgConflict", "Number of times PGMInvalidatePage() detected a mapping conflict.");
1616	PGM_REG_COUNTER(&pPGM->StatRCInvlPgSyncMonCR3, "/PGM/RC/InvlPgSyncMonitorCR3", "Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3.");
1617
1618	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1619	PGM_REG_COUNTER(&pPGM->StatTrackVirgin, "/PGM/Track/Virgin", "The number of first time shadowings");
1620	PGM_REG_COUNTER(&pPGM->StatTrackAliased, "/PGM/Track/Aliased", "The number of times switching to cRef2, i.e. the page is being shadowed by two PTs.");
1621	PGM_REG_COUNTER(&pPGM->StatTrackAliasedMany, "/PGM/Track/AliasedMany", "The number of times we're tracking using cRef2.");
1622	PGM_REG_COUNTER(&pPGM->StatTrackAliasedLots, "/PGM/Track/AliasedLots", "The number of times we're hitting pages which has overflowed cRef2");
1623	PGM_REG_COUNTER(&pPGM->StatTrackOverflows, "/PGM/Track/Overflows", "The number of times the extent list grows too long.");
1624	PGM_REG_PROFILE(&pPGM->StatTrackDeref, "/PGM/Track/Deref", "Profiling of SyncPageWorkerTrackDeref (expensive).");
1625	# endif
1626
1627	# undef PGM_REG_COUNTER
1628	# undef PGM_REG_PROFILE
1629	#endif
1630
1631	/*
1632	* Note! The layout below matches the member layout exactly!
1633	*/
1634
1635	/*
1636	* Common - stats
1637	*/
1638	for (unsigned i=0;i<pVM->cCPUs;i++)
1639	{
1640	PVMCPU pVCpu = &pVM->aCpus[i];
1641	PPGMCPU pPGM = &pVCpu->pgm.s;
1642
1643	#define PGM_REG_COUNTER(a, b, c) \
1644	rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, c, b, i); \
1645	AssertRC(rc);
1646	#define PGM_REG_PROFILE(a, b, c) \
1647	rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b, i); \
1648	AssertRC(rc);
1649
1650	PGM_REG_COUNTER(&pPGM->cGuestModeChanges, "/PGM/CPU%d/cGuestModeChanges", "Number of guest mode changes.");
1651
1652	#ifdef VBOX_WITH_STATISTICS
1653	for (unsigned j = 0; j < RT_ELEMENTS(pPGM->StatSyncPtPD); j++)
1654	STAMR3RegisterF(pVM, &pPGM->StatSyncPtPD[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
1655	"The number of SyncPT per PD n.", "/PGM/CPU%d/PDSyncPT/%04X", i, j);
1656	for (unsigned j = 0; j < RT_ELEMENTS(pPGM->StatSyncPagePD); j++)
1657	STAMR3RegisterF(pVM, &pPGM->StatSyncPagePD[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
1658	"The number of SyncPage per PD n.", "/PGM/CPU%d/PDSyncPage/%04X", i, j);
1659
1660	/* R0 only: */
1661	PGM_REG_COUNTER(&pPGM->StatR0DynMapMigrateInvlPg, "/PGM/CPU%d/R0/DynMapMigrateInvlPg", "invlpg count in PGMDynMapMigrateAutoSet.");
1662	PGM_REG_PROFILE(&pPGM->StatR0DynMapGCPageInl, "/PGM/CPU%d/R0/DynMapPageGCPageInl", "Calls to pgmR0DynMapGCPageInlined.");
1663	PGM_REG_COUNTER(&pPGM->StatR0DynMapGCPageInlHits, "/PGM/CPU%d/R0/DynMapPageGCPageInl/Hits", "Hash table lookup hits.");
1664	PGM_REG_COUNTER(&pPGM->StatR0DynMapGCPageInlMisses, "/PGM/CPU%d/R0/DynMapPageGCPageInl/Misses", "Misses that falls back to code common with PGMDynMapHCPage.");
1665	PGM_REG_COUNTER(&pPGM->StatR0DynMapGCPageInlRamHits, "/PGM/CPU%d/R0/DynMapPageGCPageInl/RamHits", "1st ram range hits.");
1666	PGM_REG_COUNTER(&pPGM->StatR0DynMapGCPageInlRamMisses, "/PGM/CPU%d/R0/DynMapPageGCPageInl/RamMisses", "1st ram range misses, takes slow path.");
1667	PGM_REG_PROFILE(&pPGM->StatR0DynMapHCPageInl, "/PGM/CPU%d/R0/DynMapPageHCPageInl", "Calls to pgmR0DynMapHCPageInlined.");
1668	PGM_REG_COUNTER(&pPGM->StatR0DynMapHCPageInlHits, "/PGM/CPU%d/R0/DynMapPageHCPageInl/Hits", "Hash table lookup hits.");
1669	PGM_REG_COUNTER(&pPGM->StatR0DynMapHCPageInlMisses, "/PGM/CPU%d/R0/DynMapPageHCPageInl/Misses", "Misses that falls back to code common with PGMDynMapHCPage.");
1670	PGM_REG_COUNTER(&pPGM->StatR0DynMapPage, "/PGM/CPU%d/R0/DynMapPage", "Calls to pgmR0DynMapPage");
1671	PGM_REG_COUNTER(&pPGM->StatR0DynMapSetOptimize, "/PGM/CPU%d/R0/DynMapPage/SetOptimize", "Calls to pgmDynMapOptimizeAutoSet.");
1672	PGM_REG_COUNTER(&pPGM->StatR0DynMapSetSearchFlushes, "/PGM/CPU%d/R0/DynMapPage/SetSearchFlushes","Set search restorting to subset flushes.");
1673	PGM_REG_COUNTER(&pPGM->StatR0DynMapSetSearchHits, "/PGM/CPU%d/R0/DynMapPage/SetSearchHits", "Set search hits.");
1674	PGM_REG_COUNTER(&pPGM->StatR0DynMapSetSearchMisses, "/PGM/CPU%d/R0/DynMapPage/SetSearchMisses", "Set search misses.");
1675	PGM_REG_PROFILE(&pPGM->StatR0DynMapHCPage, "/PGM/CPU%d/R0/DynMapPage/HCPage", "Calls to PGMDynMapHCPage (ring-0).");
1676	PGM_REG_COUNTER(&pPGM->StatR0DynMapPageHits0, "/PGM/CPU%d/R0/DynMapPage/Hits0", "Hits at iPage+0");
1677	PGM_REG_COUNTER(&pPGM->StatR0DynMapPageHits1, "/PGM/CPU%d/R0/DynMapPage/Hits1", "Hits at iPage+1");
1678	PGM_REG_COUNTER(&pPGM->StatR0DynMapPageHits2, "/PGM/CPU%d/R0/DynMapPage/Hits2", "Hits at iPage+2");
1679	PGM_REG_COUNTER(&pPGM->StatR0DynMapPageInvlPg, "/PGM/CPU%d/R0/DynMapPage/InvlPg", "invlpg count in pgmR0DynMapPageSlow.");
1680	PGM_REG_COUNTER(&pPGM->StatR0DynMapPageSlow, "/PGM/CPU%d/R0/DynMapPage/Slow", "Calls to pgmR0DynMapPageSlow - subtract this from pgmR0DynMapPage to get 1st level hits.");
1681	PGM_REG_COUNTER(&pPGM->StatR0DynMapPageSlowLoopHits, "/PGM/CPU%d/R0/DynMapPage/SlowLoopHits" , "Hits in the loop path.");
1682	PGM_REG_COUNTER(&pPGM->StatR0DynMapPageSlowLoopMisses, "/PGM/CPU%d/R0/DynMapPage/SlowLoopMisses", "Misses in the loop path. NonLoopMisses = Slow - SlowLoopHit - SlowLoopMisses");
1683	//PGM_REG_COUNTER(&pPGM->StatR0DynMapPageSlowLostHits, "/PGM/CPU%d/R0/DynMapPage/SlowLostHits", "Lost hits.");
1684	PGM_REG_COUNTER(&pPGM->StatR0DynMapSubsets, "/PGM/CPU%d/R0/Subsets", "Times PGMDynMapPushAutoSubset was called.");
1685	PGM_REG_COUNTER(&pPGM->StatR0DynMapPopFlushes, "/PGM/CPU%d/R0/SubsetPopFlushes", "Times PGMDynMapPopAutoSubset flushes the subset.");
1686	PGM_REG_COUNTER(&pPGM->aStatR0DynMapSetSize[0], "/PGM/CPU%d/R0/SetSize000..09", "00-09% filled");
1687	PGM_REG_COUNTER(&pPGM->aStatR0DynMapSetSize[1], "/PGM/CPU%d/R0/SetSize010..19", "10-19% filled");
1688	PGM_REG_COUNTER(&pPGM->aStatR0DynMapSetSize[2], "/PGM/CPU%d/R0/SetSize020..29", "20-29% filled");
1689	PGM_REG_COUNTER(&pPGM->aStatR0DynMapSetSize[3], "/PGM/CPU%d/R0/SetSize030..39", "30-39% filled");
1690	PGM_REG_COUNTER(&pPGM->aStatR0DynMapSetSize[4], "/PGM/CPU%d/R0/SetSize040..49", "40-49% filled");
1691	PGM_REG_COUNTER(&pPGM->aStatR0DynMapSetSize[5], "/PGM/CPU%d/R0/SetSize050..59", "50-59% filled");
1692	PGM_REG_COUNTER(&pPGM->aStatR0DynMapSetSize[6], "/PGM/CPU%d/R0/SetSize060..69", "60-69% filled");
1693	PGM_REG_COUNTER(&pPGM->aStatR0DynMapSetSize[7], "/PGM/CPU%d/R0/SetSize070..79", "70-79% filled");
1694	PGM_REG_COUNTER(&pPGM->aStatR0DynMapSetSize[8], "/PGM/CPU%d/R0/SetSize080..89", "80-89% filled");
1695	PGM_REG_COUNTER(&pPGM->aStatR0DynMapSetSize[9], "/PGM/CPU%d/R0/SetSize090..99", "90-99% filled");
1696	PGM_REG_COUNTER(&pPGM->aStatR0DynMapSetSize[10], "/PGM/CPU%d/R0/SetSize100", "100% filled");
1697
1698	/* RZ only: */
1699	PGM_REG_PROFILE(&pPGM->StatRZTrap0e, "/PGM/CPU%d/RZ/Trap0e", "Profiling of the PGMTrap0eHandler() body.");
1700	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTimeCheckPageFault, "/PGM/CPU%d/RZ/Trap0e/Time/CheckPageFault", "Profiling of checking for dirty/access emulation faults.");
1701	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTimeSyncPT, "/PGM/CPU%d/RZ/Trap0e/Time/SyncPT", "Profiling of lazy page table syncing.");
1702	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTimeMapping, "/PGM/CPU%d/RZ/Trap0e/Time/Mapping", "Profiling of checking virtual mappings.");
1703	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTimeOutOfSync, "/PGM/CPU%d/RZ/Trap0e/Time/OutOfSync", "Profiling of out of sync page handling.");
1704	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTimeHandlers, "/PGM/CPU%d/RZ/Trap0e/Time/Handlers", "Profiling of checking handlers.");
1705	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTime2CSAM, "/PGM/CPU%d/RZ/Trap0e/Time2/CSAM", "Profiling of the Trap0eHandler body when the cause is CSAM.");
1706	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTime2DirtyAndAccessed, "/PGM/CPU%d/RZ/Trap0e/Time2/DirtyAndAccessedBits", "Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation.");
1707	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTime2GuestTrap, "/PGM/CPU%d/RZ/Trap0e/Time2/GuestTrap", "Profiling of the Trap0eHandler body when the cause is a guest trap.");
1708	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTime2HndPhys, "/PGM/CPU%d/RZ/Trap0e/Time2/HandlerPhysical", "Profiling of the Trap0eHandler body when the cause is a physical handler.");
1709	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTime2HndVirt, "/PGM/CPU%d/RZ/Trap0e/Time2/HandlerVirtual", "Profiling of the Trap0eHandler body when the cause is a virtual handler.");
1710	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTime2HndUnhandled, "/PGM/CPU%d/RZ/Trap0e/Time2/HandlerUnhandled", "Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page.");
1711	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTime2Misc, "/PGM/CPU%d/RZ/Trap0e/Time2/Misc", "Profiling of the Trap0eHandler body when the cause is not known.");
1712	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTime2OutOfSync, "/PGM/CPU%d/RZ/Trap0e/Time2/OutOfSync", "Profiling of the Trap0eHandler body when the cause is an out-of-sync page.");
1713	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTime2OutOfSyncHndPhys, "/PGM/CPU%d/RZ/Trap0e/Time2/OutOfSyncHndPhys", "Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page.");
1714	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTime2OutOfSyncHndVirt, "/PGM/CPU%d/RZ/Trap0e/Time2/OutOfSyncHndVirt", "Profiling of the Trap0eHandler body when the cause is an out-of-sync virtual handler page.");
1715	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTime2OutOfSyncHndObs, "/PGM/CPU%d/RZ/Trap0e/Time2/OutOfSyncObsHnd", "Profiling of the Trap0eHandler body when the cause is an obsolete handler page.");
1716	PGM_REG_PROFILE(&pPGM->StatRZTrap0eTime2SyncPT, "/PGM/CPU%d/RZ/Trap0e/Time2/SyncPT", "Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT.");
1717	PGM_REG_COUNTER(&pPGM->StatRZTrap0eConflicts, "/PGM/CPU%d/RZ/Trap0e/Conflicts", "The number of times #PF was caused by an undetected conflict.");
1718	PGM_REG_COUNTER(&pPGM->StatRZTrap0eHandlersMapping, "/PGM/CPU%d/RZ/Trap0e/Handlers/Mapping", "Number of traps due to access handlers in mappings.");
1719	PGM_REG_COUNTER(&pPGM->StatRZTrap0eHandlersOutOfSync, "/PGM/CPU%d/RZ/Trap0e/Handlers/OutOfSync", "Number of traps due to out-of-sync handled pages.");
1720	PGM_REG_COUNTER(&pPGM->StatRZTrap0eHandlersPhysical, "/PGM/CPU%d/RZ/Trap0e/Handlers/Physical", "Number of traps due to physical access handlers.");
1721	PGM_REG_COUNTER(&pPGM->StatRZTrap0eHandlersVirtual, "/PGM/CPU%d/RZ/Trap0e/Handlers/Virtual", "Number of traps due to virtual access handlers.");
1722	PGM_REG_COUNTER(&pPGM->StatRZTrap0eHandlersVirtualByPhys, "/PGM/CPU%d/RZ/Trap0e/Handlers/VirtualByPhys", "Number of traps due to virtual access handlers by physical address.");
1723	PGM_REG_COUNTER(&pPGM->StatRZTrap0eHandlersVirtualUnmarked,"/PGM/CPU%d/RZ/Trap0e/Handlers/VirtualUnmarked","Number of traps due to virtual access handlers by virtual address (without proper physical flags).");
1724	PGM_REG_COUNTER(&pPGM->StatRZTrap0eHandlersUnhandled, "/PGM/CPU%d/RZ/Trap0e/Handlers/Unhandled", "Number of traps due to access outside range of monitored page(s).");
1725	PGM_REG_COUNTER(&pPGM->StatRZTrap0eHandlersInvalid, "/PGM/CPU%d/RZ/Trap0e/Handlers/Invalid", "Number of traps due to access to invalid physical memory.");
1726	PGM_REG_COUNTER(&pPGM->StatRZTrap0eUSNotPresentRead, "/PGM/CPU%d/RZ/Trap0e/Err/User/NPRead", "Number of user mode not present read page faults.");
1727	PGM_REG_COUNTER(&pPGM->StatRZTrap0eUSNotPresentWrite, "/PGM/CPU%d/RZ/Trap0e/Err/User/NPWrite", "Number of user mode not present write page faults.");
1728	PGM_REG_COUNTER(&pPGM->StatRZTrap0eUSWrite, "/PGM/CPU%d/RZ/Trap0e/Err/User/Write", "Number of user mode write page faults.");
1729	PGM_REG_COUNTER(&pPGM->StatRZTrap0eUSReserved, "/PGM/CPU%d/RZ/Trap0e/Err/User/Reserved", "Number of user mode reserved bit page faults.");
1730	PGM_REG_COUNTER(&pPGM->StatRZTrap0eUSNXE, "/PGM/CPU%d/RZ/Trap0e/Err/User/NXE", "Number of user mode NXE page faults.");
1731	PGM_REG_COUNTER(&pPGM->StatRZTrap0eUSRead, "/PGM/CPU%d/RZ/Trap0e/Err/User/Read", "Number of user mode read page faults.");
1732	PGM_REG_COUNTER(&pPGM->StatRZTrap0eSVNotPresentRead, "/PGM/CPU%d/RZ/Trap0e/Err/Supervisor/NPRead", "Number of supervisor mode not present read page faults.");
1733	PGM_REG_COUNTER(&pPGM->StatRZTrap0eSVNotPresentWrite, "/PGM/CPU%d/RZ/Trap0e/Err/Supervisor/NPWrite", "Number of supervisor mode not present write page faults.");
1734	PGM_REG_COUNTER(&pPGM->StatRZTrap0eSVWrite, "/PGM/CPU%d/RZ/Trap0e/Err/Supervisor/Write", "Number of supervisor mode write page faults.");
1735	PGM_REG_COUNTER(&pPGM->StatRZTrap0eSVReserved, "/PGM/CPU%d/RZ/Trap0e/Err/Supervisor/Reserved", "Number of supervisor mode reserved bit page faults.");
1736	PGM_REG_COUNTER(&pPGM->StatRZTrap0eSNXE, "/PGM/CPU%d/RZ/Trap0e/Err/Supervisor/NXE", "Number of supervisor mode NXE page faults.");
1737	PGM_REG_COUNTER(&pPGM->StatRZTrap0eGuestPF, "/PGM/CPU%d/RZ/Trap0e/GuestPF", "Number of real guest page faults.");
1738	PGM_REG_COUNTER(&pPGM->StatRZTrap0eGuestPFUnh, "/PGM/CPU%d/RZ/Trap0e/GuestPF/Unhandled", "Number of real guest page faults from the 'unhandled' case.");
1739	PGM_REG_COUNTER(&pPGM->StatRZTrap0eGuestPFMapping, "/PGM/CPU%d/RZ/Trap0e/GuestPF/InMapping", "Number of real guest page faults in a mapping.");
1740	PGM_REG_COUNTER(&pPGM->StatRZTrap0eWPEmulInRZ, "/PGM/CPU%d/RZ/Trap0e/WP/InRZ", "Number of guest page faults due to X86_CR0_WP emulation.");
1741	PGM_REG_COUNTER(&pPGM->StatRZTrap0eWPEmulToR3, "/PGM/CPU%d/RZ/Trap0e/WP/ToR3", "Number of guest page faults due to X86_CR0_WP emulation (forward to R3 for emulation).");
1742	for (unsigned j = 0; j < RT_ELEMENTS(pPGM->StatRZTrap0ePD); j++)
1743	STAMR3RegisterF(pVM, &pPGM->StatRZTrap0ePD[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
1744	"The number of traps in page directory n.", "/PGM/CPU%d/RZ/Trap0e/PD/%04X", i, j);
1745
1746	PGM_REG_COUNTER(&pPGM->StatRZGuestCR3WriteHandled, "/PGM/CPU%d/RZ/CR3WriteHandled", "The number of times the Guest CR3 change was successfully handled.");
1747	PGM_REG_COUNTER(&pPGM->StatRZGuestCR3WriteUnhandled, "/PGM/CPU%d/RZ/CR3WriteUnhandled", "The number of times the Guest CR3 change was passed back to the recompiler.");
1748	PGM_REG_COUNTER(&pPGM->StatRZGuestCR3WriteConflict, "/PGM/CPU%d/RZ/CR3WriteConflict", "The number of times the Guest CR3 monitoring detected a conflict.");
1749	PGM_REG_COUNTER(&pPGM->StatRZGuestROMWriteHandled, "/PGM/CPU%d/RZ/ROMWriteHandled", "The number of times the Guest ROM change was successfully handled.");
1750	PGM_REG_COUNTER(&pPGM->StatRZGuestROMWriteUnhandled, "/PGM/CPU%d/RZ/ROMWriteUnhandled", "The number of times the Guest ROM change was passed back to the recompiler.");
1751
1752	/* HC only: */
1753
1754	/* RZ & R3: */
1755	PGM_REG_PROFILE(&pPGM->StatRZSyncCR3, "/PGM/CPU%d/RZ/SyncCR3", "Profiling of the PGMSyncCR3() body.");
1756	PGM_REG_PROFILE(&pPGM->StatRZSyncCR3Handlers, "/PGM/CPU%d/RZ/SyncCR3/Handlers", "Profiling of the PGMSyncCR3() update handler section.");
1757	PGM_REG_COUNTER(&pPGM->StatRZSyncCR3Global, "/PGM/CPU%d/RZ/SyncCR3/Global", "The number of global CR3 syncs.");
1758	PGM_REG_COUNTER(&pPGM->StatRZSyncCR3NotGlobal, "/PGM/CPU%d/RZ/SyncCR3/NotGlobal", "The number of non-global CR3 syncs.");
1759	PGM_REG_COUNTER(&pPGM->StatRZSyncCR3DstCacheHit, "/PGM/CPU%d/RZ/SyncCR3/DstChacheHit", "The number of times we got some kind of a cache hit.");
1760	PGM_REG_COUNTER(&pPGM->StatRZSyncCR3DstFreed, "/PGM/CPU%d/RZ/SyncCR3/DstFreed", "The number of times we've had to free a shadow entry.");
1761	PGM_REG_COUNTER(&pPGM->StatRZSyncCR3DstFreedSrcNP, "/PGM/CPU%d/RZ/SyncCR3/DstFreedSrcNP", "The number of times we've had to free a shadow entry for which the source entry was not present.");
1762	PGM_REG_COUNTER(&pPGM->StatRZSyncCR3DstNotPresent, "/PGM/CPU%d/RZ/SyncCR3/DstNotPresent", "The number of times we've encountered a not present shadow entry for a present guest entry.");
1763	PGM_REG_COUNTER(&pPGM->StatRZSyncCR3DstSkippedGlobalPD, "/PGM/CPU%d/RZ/SyncCR3/DstSkippedGlobalPD", "The number of times a global page directory wasn't flushed.");
1764	PGM_REG_COUNTER(&pPGM->StatRZSyncCR3DstSkippedGlobalPT, "/PGM/CPU%d/RZ/SyncCR3/DstSkippedGlobalPT", "The number of times a page table with only global entries wasn't flushed.");
1765	PGM_REG_PROFILE(&pPGM->StatRZSyncPT, "/PGM/CPU%d/RZ/SyncPT", "Profiling of the pfnSyncPT() body.");
1766	PGM_REG_COUNTER(&pPGM->StatRZSyncPTFailed, "/PGM/CPU%d/RZ/SyncPT/Failed", "The number of times pfnSyncPT() failed.");
1767	PGM_REG_COUNTER(&pPGM->StatRZSyncPT4K, "/PGM/CPU%d/RZ/SyncPT/4K", "Nr of 4K PT syncs");
1768	PGM_REG_COUNTER(&pPGM->StatRZSyncPT4M, "/PGM/CPU%d/RZ/SyncPT/4M", "Nr of 4M PT syncs");
1769	PGM_REG_COUNTER(&pPGM->StatRZSyncPagePDNAs, "/PGM/CPU%d/RZ/SyncPagePDNAs", "The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit.");
1770	PGM_REG_COUNTER(&pPGM->StatRZSyncPagePDOutOfSync, "/PGM/CPU%d/RZ/SyncPagePDOutOfSync", "The number of time we've encountered an out-of-sync PD in SyncPage.");
1771	PGM_REG_COUNTER(&pPGM->StatRZAccessedPage, "/PGM/CPU%d/RZ/AccessedPage", "The number of pages marked not present for accessed bit emulation.");
1772	PGM_REG_PROFILE(&pPGM->StatRZDirtyBitTracking, "/PGM/CPU%d/RZ/DirtyPage", "Profiling the dirty bit tracking in CheckPageFault().");
1773	PGM_REG_COUNTER(&pPGM->StatRZDirtyPage, "/PGM/CPU%d/RZ/DirtyPage/Mark", "The number of pages marked read-only for dirty bit tracking.");
1774	PGM_REG_COUNTER(&pPGM->StatRZDirtyPageBig, "/PGM/CPU%d/RZ/DirtyPage/MarkBig", "The number of 4MB pages marked read-only for dirty bit tracking.");
1775	PGM_REG_COUNTER(&pPGM->StatRZDirtyPageSkipped, "/PGM/CPU%d/RZ/DirtyPage/Skipped", "The number of pages already dirty or readonly.");
1776	PGM_REG_COUNTER(&pPGM->StatRZDirtyPageTrap, "/PGM/CPU%d/RZ/DirtyPage/Trap", "The number of traps generated for dirty bit tracking.");
1777	PGM_REG_COUNTER(&pPGM->StatRZDirtyPageStale, "/PGM/CPU%d/RZ/DirtyPage/Stale", "The number of traps generated for dirty bit tracking (stale tlb entries).");
1778	PGM_REG_COUNTER(&pPGM->StatRZDirtiedPage, "/PGM/CPU%d/RZ/DirtyPage/SetDirty", "The number of pages marked dirty because of write accesses.");
1779	PGM_REG_COUNTER(&pPGM->StatRZDirtyTrackRealPF, "/PGM/CPU%d/RZ/DirtyPage/RealPF", "The number of real pages faults during dirty bit tracking.");
1780	PGM_REG_COUNTER(&pPGM->StatRZPageAlreadyDirty, "/PGM/CPU%d/RZ/DirtyPage/AlreadySet", "The number of pages already marked dirty because of write accesses.");
1781	PGM_REG_PROFILE(&pPGM->StatRZInvalidatePage, "/PGM/CPU%d/RZ/InvalidatePage", "PGMInvalidatePage() profiling.");
1782	PGM_REG_COUNTER(&pPGM->StatRZInvalidatePage4KBPages, "/PGM/CPU%d/RZ/InvalidatePage/4KBPages", "The number of times PGMInvalidatePage() was called for a 4KB page.");
1783	PGM_REG_COUNTER(&pPGM->StatRZInvalidatePage4MBPages, "/PGM/CPU%d/RZ/InvalidatePage/4MBPages", "The number of times PGMInvalidatePage() was called for a 4MB page.");
1784	PGM_REG_COUNTER(&pPGM->StatRZInvalidatePage4MBPagesSkip, "/PGM/CPU%d/RZ/InvalidatePage/4MBPagesSkip","The number of times PGMInvalidatePage() skipped a 4MB page.");
1785	PGM_REG_COUNTER(&pPGM->StatRZInvalidatePagePDMappings, "/PGM/CPU%d/RZ/InvalidatePage/PDMappings", "The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict).");
1786	PGM_REG_COUNTER(&pPGM->StatRZInvalidatePagePDNAs, "/PGM/CPU%d/RZ/InvalidatePage/PDNAs", "The number of times PGMInvalidatePage() was called for a not accessed page directory.");
1787	PGM_REG_COUNTER(&pPGM->StatRZInvalidatePagePDNPs, "/PGM/CPU%d/RZ/InvalidatePage/PDNPs", "The number of times PGMInvalidatePage() was called for a not present page directory.");
1788	PGM_REG_COUNTER(&pPGM->StatRZInvalidatePagePDOutOfSync, "/PGM/CPU%d/RZ/InvalidatePage/PDOutOfSync", "The number of times PGMInvalidatePage() was called for an out of sync page directory.");
1789	PGM_REG_COUNTER(&pPGM->StatRZInvalidatePageSkipped, "/PGM/CPU%d/RZ/InvalidatePage/Skipped", "The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3.");
1790	PGM_REG_COUNTER(&pPGM->StatRZPageOutOfSyncSupervisor, "/PGM/CPU%d/RZ/OutOfSync/SuperVisor", "Number of traps due to pages out of sync and times VerifyAccessSyncPage calls SyncPage.");
1791	PGM_REG_COUNTER(&pPGM->StatRZPageOutOfSyncUser, "/PGM/CPU%d/RZ/OutOfSync/User", "Number of traps due to pages out of sync and times VerifyAccessSyncPage calls SyncPage.");
1792	PGM_REG_PROFILE(&pPGM->StatRZPrefetch, "/PGM/CPU%d/RZ/Prefetch", "PGMPrefetchPage profiling.");
1793	PGM_REG_PROFILE(&pPGM->StatRZFlushTLB, "/PGM/CPU%d/RZ/FlushTLB", "Profiling of the PGMFlushTLB() body.");
1794	PGM_REG_COUNTER(&pPGM->StatRZFlushTLBNewCR3, "/PGM/CPU%d/RZ/FlushTLB/NewCR3", "The number of times PGMFlushTLB was called with a new CR3, non-global. (switch)");
1795	PGM_REG_COUNTER(&pPGM->StatRZFlushTLBNewCR3Global, "/PGM/CPU%d/RZ/FlushTLB/NewCR3Global", "The number of times PGMFlushTLB was called with a new CR3, global. (switch)");
1796	PGM_REG_COUNTER(&pPGM->StatRZFlushTLBSameCR3, "/PGM/CPU%d/RZ/FlushTLB/SameCR3", "The number of times PGMFlushTLB was called with the same CR3, non-global. (flush)");
1797	PGM_REG_COUNTER(&pPGM->StatRZFlushTLBSameCR3Global, "/PGM/CPU%d/RZ/FlushTLB/SameCR3Global", "The number of times PGMFlushTLB was called with the same CR3, global. (flush)");
1798	PGM_REG_PROFILE(&pPGM->StatRZGstModifyPage, "/PGM/CPU%d/RZ/GstModifyPage", "Profiling of the PGMGstModifyPage() body.");
1799
1800	PGM_REG_PROFILE(&pPGM->StatR3SyncCR3, "/PGM/CPU%d/R3/SyncCR3", "Profiling of the PGMSyncCR3() body.");
1801	PGM_REG_PROFILE(&pPGM->StatR3SyncCR3Handlers, "/PGM/CPU%d/R3/SyncCR3/Handlers", "Profiling of the PGMSyncCR3() update handler section.");
1802	PGM_REG_COUNTER(&pPGM->StatR3SyncCR3Global, "/PGM/CPU%d/R3/SyncCR3/Global", "The number of global CR3 syncs.");
1803	PGM_REG_COUNTER(&pPGM->StatR3SyncCR3NotGlobal, "/PGM/CPU%d/R3/SyncCR3/NotGlobal", "The number of non-global CR3 syncs.");
1804	PGM_REG_COUNTER(&pPGM->StatR3SyncCR3DstCacheHit, "/PGM/CPU%d/R3/SyncCR3/DstChacheHit", "The number of times we got some kind of a cache hit.");
1805	PGM_REG_COUNTER(&pPGM->StatR3SyncCR3DstFreed, "/PGM/CPU%d/R3/SyncCR3/DstFreed", "The number of times we've had to free a shadow entry.");
1806	PGM_REG_COUNTER(&pPGM->StatR3SyncCR3DstFreedSrcNP, "/PGM/CPU%d/R3/SyncCR3/DstFreedSrcNP", "The number of times we've had to free a shadow entry for which the source entry was not present.");
1807	PGM_REG_COUNTER(&pPGM->StatR3SyncCR3DstNotPresent, "/PGM/CPU%d/R3/SyncCR3/DstNotPresent", "The number of times we've encountered a not present shadow entry for a present guest entry.");
1808	PGM_REG_COUNTER(&pPGM->StatR3SyncCR3DstSkippedGlobalPD, "/PGM/CPU%d/R3/SyncCR3/DstSkippedGlobalPD", "The number of times a global page directory wasn't flushed.");
1809	PGM_REG_COUNTER(&pPGM->StatR3SyncCR3DstSkippedGlobalPT, "/PGM/CPU%d/R3/SyncCR3/DstSkippedGlobalPT", "The number of times a page table with only global entries wasn't flushed.");
1810	PGM_REG_PROFILE(&pPGM->StatR3SyncPT, "/PGM/CPU%d/R3/SyncPT", "Profiling of the pfnSyncPT() body.");
1811	PGM_REG_COUNTER(&pPGM->StatR3SyncPTFailed, "/PGM/CPU%d/R3/SyncPT/Failed", "The number of times pfnSyncPT() failed.");
1812	PGM_REG_COUNTER(&pPGM->StatR3SyncPT4K, "/PGM/CPU%d/R3/SyncPT/4K", "Nr of 4K PT syncs");
1813	PGM_REG_COUNTER(&pPGM->StatR3SyncPT4M, "/PGM/CPU%d/R3/SyncPT/4M", "Nr of 4M PT syncs");
1814	PGM_REG_COUNTER(&pPGM->StatR3SyncPagePDNAs, "/PGM/CPU%d/R3/SyncPagePDNAs", "The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit.");
1815	PGM_REG_COUNTER(&pPGM->StatR3SyncPagePDOutOfSync, "/PGM/CPU%d/R3/SyncPagePDOutOfSync", "The number of time we've encountered an out-of-sync PD in SyncPage.");
1816	PGM_REG_COUNTER(&pPGM->StatR3AccessedPage, "/PGM/CPU%d/R3/AccessedPage", "The number of pages marked not present for accessed bit emulation.");
1817	PGM_REG_PROFILE(&pPGM->StatR3DirtyBitTracking, "/PGM/CPU%d/R3/DirtyPage", "Profiling the dirty bit tracking in CheckPageFault().");
1818	PGM_REG_COUNTER(&pPGM->StatR3DirtyPage, "/PGM/CPU%d/R3/DirtyPage/Mark", "The number of pages marked read-only for dirty bit tracking.");
1819	PGM_REG_COUNTER(&pPGM->StatR3DirtyPageBig, "/PGM/CPU%d/R3/DirtyPage/MarkBig", "The number of 4MB pages marked read-only for dirty bit tracking.");
1820	PGM_REG_COUNTER(&pPGM->StatR3DirtyPageSkipped, "/PGM/CPU%d/R3/DirtyPage/Skipped", "The number of pages already dirty or readonly.");
1821	PGM_REG_COUNTER(&pPGM->StatR3DirtyPageTrap, "/PGM/CPU%d/R3/DirtyPage/Trap", "The number of traps generated for dirty bit tracking.");
1822	PGM_REG_COUNTER(&pPGM->StatR3DirtiedPage, "/PGM/CPU%d/R3/DirtyPage/SetDirty", "The number of pages marked dirty because of write accesses.");
1823	PGM_REG_COUNTER(&pPGM->StatR3DirtyTrackRealPF, "/PGM/CPU%d/R3/DirtyPage/RealPF", "The number of real pages faults during dirty bit tracking.");
1824	PGM_REG_COUNTER(&pPGM->StatR3PageAlreadyDirty, "/PGM/CPU%d/R3/DirtyPage/AlreadySet", "The number of pages already marked dirty because of write accesses.");
1825	PGM_REG_PROFILE(&pPGM->StatR3InvalidatePage, "/PGM/CPU%d/R3/InvalidatePage", "PGMInvalidatePage() profiling.");
1826	PGM_REG_COUNTER(&pPGM->StatR3InvalidatePage4KBPages, "/PGM/CPU%d/R3/InvalidatePage/4KBPages", "The number of times PGMInvalidatePage() was called for a 4KB page.");
1827	PGM_REG_COUNTER(&pPGM->StatR3InvalidatePage4MBPages, "/PGM/CPU%d/R3/InvalidatePage/4MBPages", "The number of times PGMInvalidatePage() was called for a 4MB page.");
1828	PGM_REG_COUNTER(&pPGM->StatR3InvalidatePage4MBPagesSkip, "/PGM/CPU%d/R3/InvalidatePage/4MBPagesSkip","The number of times PGMInvalidatePage() skipped a 4MB page.");
1829	PGM_REG_COUNTER(&pPGM->StatR3InvalidatePagePDMappings, "/PGM/CPU%d/R3/InvalidatePage/PDMappings", "The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict).");
1830	PGM_REG_COUNTER(&pPGM->StatR3InvalidatePagePDNAs, "/PGM/CPU%d/R3/InvalidatePage/PDNAs", "The number of times PGMInvalidatePage() was called for a not accessed page directory.");
1831	PGM_REG_COUNTER(&pPGM->StatR3InvalidatePagePDNPs, "/PGM/CPU%d/R3/InvalidatePage/PDNPs", "The number of times PGMInvalidatePage() was called for a not present page directory.");
1832	PGM_REG_COUNTER(&pPGM->StatR3InvalidatePagePDOutOfSync, "/PGM/CPU%d/R3/InvalidatePage/PDOutOfSync", "The number of times PGMInvalidatePage() was called for an out of sync page directory.");
1833	PGM_REG_COUNTER(&pPGM->StatR3InvalidatePageSkipped, "/PGM/CPU%d/R3/InvalidatePage/Skipped", "The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3.");
1834	PGM_REG_COUNTER(&pPGM->StatR3PageOutOfSyncSupervisor, "/PGM/CPU%d/R3/OutOfSync/SuperVisor", "Number of traps due to pages out of sync and times VerifyAccessSyncPage calls SyncPage.");
1835	PGM_REG_COUNTER(&pPGM->StatR3PageOutOfSyncUser, "/PGM/CPU%d/R3/OutOfSync/User", "Number of traps due to pages out of sync and times VerifyAccessSyncPage calls SyncPage.");
1836	PGM_REG_PROFILE(&pPGM->StatR3Prefetch, "/PGM/CPU%d/R3/Prefetch", "PGMPrefetchPage profiling.");
1837	PGM_REG_PROFILE(&pPGM->StatR3FlushTLB, "/PGM/CPU%d/R3/FlushTLB", "Profiling of the PGMFlushTLB() body.");
1838	PGM_REG_COUNTER(&pPGM->StatR3FlushTLBNewCR3, "/PGM/CPU%d/R3/FlushTLB/NewCR3", "The number of times PGMFlushTLB was called with a new CR3, non-global. (switch)");
1839	PGM_REG_COUNTER(&pPGM->StatR3FlushTLBNewCR3Global, "/PGM/CPU%d/R3/FlushTLB/NewCR3Global", "The number of times PGMFlushTLB was called with a new CR3, global. (switch)");
1840	PGM_REG_COUNTER(&pPGM->StatR3FlushTLBSameCR3, "/PGM/CPU%d/R3/FlushTLB/SameCR3", "The number of times PGMFlushTLB was called with the same CR3, non-global. (flush)");
1841	PGM_REG_COUNTER(&pPGM->StatR3FlushTLBSameCR3Global, "/PGM/CPU%d/R3/FlushTLB/SameCR3Global", "The number of times PGMFlushTLB was called with the same CR3, global. (flush)");
1842	PGM_REG_PROFILE(&pPGM->StatR3GstModifyPage, "/PGM/CPU%d/R3/GstModifyPage", "Profiling of the PGMGstModifyPage() body.");
1843	#endif /* VBOX_WITH_STATISTICS */
1844
1845	#undef PGM_REG_PROFILE
1846	#undef PGM_REG_COUNTER
1847
1848	}
1849	}
1850
1851
1852	/**
1853	* Init the PGM bits that rely on VMMR0 and MM to be fully initialized.
1854	*
1855	* The dynamic mapping area will also be allocated and initialized at this
1856	* time. We could allocate it during PGMR3Init of course, but the mapping
1857	* wouldn't be allocated at that time preventing us from setting up the
1858	* page table entries with the dummy page.
1859	*
1860	* @returns VBox status code.
1861	* @param pVM VM handle.
1862	*/
1863	VMMR3DECL(int) PGMR3InitDynMap(PVM pVM)
1864	{
1865	RTGCPTR GCPtr;
1866	int rc;
1867
1868	/*
1869	* Reserve space for the dynamic mappings.
1870	*/
1871	rc = MMR3HyperReserve(pVM, MM_HYPER_DYNAMIC_SIZE, "Dynamic mapping", &GCPtr);
1872	if (RT_SUCCESS(rc))
1873	pVM->pgm.s.pbDynPageMapBaseGC = GCPtr;
1874
1875	if ( RT_SUCCESS(rc)
1876	&& (pVM->pgm.s.pbDynPageMapBaseGC >> X86_PD_PAE_SHIFT) != ((pVM->pgm.s.pbDynPageMapBaseGC + MM_HYPER_DYNAMIC_SIZE - 1) >> X86_PD_PAE_SHIFT))
1877	{
1878	rc = MMR3HyperReserve(pVM, MM_HYPER_DYNAMIC_SIZE, "Dynamic mapping not crossing", &GCPtr);
1879	if (RT_SUCCESS(rc))
1880	pVM->pgm.s.pbDynPageMapBaseGC = GCPtr;
1881	}
1882	if (RT_SUCCESS(rc))
1883	{
1884	AssertRelease((pVM->pgm.s.pbDynPageMapBaseGC >> X86_PD_PAE_SHIFT) == ((pVM->pgm.s.pbDynPageMapBaseGC + MM_HYPER_DYNAMIC_SIZE - 1) >> X86_PD_PAE_SHIFT));
1885	MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
1886	}
1887	return rc;
1888	}
1889
1890
1891	/**
1892	* Ring-3 init finalizing.
1893	*
1894	* @returns VBox status code.
1895	* @param pVM The VM handle.
1896	*/
1897	VMMR3DECL(int) PGMR3InitFinalize(PVM pVM)
1898	{
1899	int rc;
1900
1901	/*
1902	* Reserve space for the dynamic mappings.
1903	* Initialize the dynamic mapping pages with dummy pages to simply the cache.
1904	*/
1905	/* get the pointer to the page table entries. */
1906	PPGMMAPPING pMapping = pgmGetMapping(pVM, pVM->pgm.s.pbDynPageMapBaseGC);
1907	AssertRelease(pMapping);
1908	const uintptr_t off = pVM->pgm.s.pbDynPageMapBaseGC - pMapping->GCPtr;
1909	const unsigned iPT = off >> X86_PD_SHIFT;
1910	const unsigned iPG = (off >> X86_PT_SHIFT) & X86_PT_MASK;
1911	pVM->pgm.s.paDynPageMap32BitPTEsGC = pMapping->aPTs[iPT].pPTRC + iPG * sizeof(pMapping->aPTs[0].pPTR3->a[0]);
1912	pVM->pgm.s.paDynPageMapPaePTEsGC = pMapping->aPTs[iPT].paPaePTsRC + iPG * sizeof(pMapping->aPTs[0].paPaePTsR3->a[0]);
1913
1914	/* init cache */
1915	RTHCPHYS HCPhysDummy = MMR3PageDummyHCPhys(pVM);
1916	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHCPhysDynPageMapCache); i++)
1917	pVM->pgm.s.aHCPhysDynPageMapCache[i] = HCPhysDummy;
1918
1919	for (unsigned i = 0; i < MM_HYPER_DYNAMIC_SIZE; i += PAGE_SIZE)
1920	{
1921	rc = PGMMap(pVM, pVM->pgm.s.pbDynPageMapBaseGC + i, HCPhysDummy, PAGE_SIZE, 0);
1922	AssertRCReturn(rc, rc);
1923	}
1924
1925	/*
1926	* Note that AMD uses all the 8 reserved bits for the address (so 40 bits in total);
1927	* Intel only goes up to 36 bits, so we stick to 36 as well.
1928	*/
1929	/** @todo How to test for the 40 bits support? Long mode seems to be the test criterium. */
1930	uint32_t u32Dummy, u32Features;
1931	CPUMGetGuestCpuId(VMMGetCpu(pVM), 1, &u32Dummy, &u32Dummy, &u32Dummy, &u32Features);
1932
1933	if (u32Features & X86_CPUID_FEATURE_EDX_PSE36)
1934	pVM->pgm.s.GCPhys4MBPSEMask = RT_BIT_64(36) - 1;
1935	else
1936	pVM->pgm.s.GCPhys4MBPSEMask = RT_BIT_64(32) - 1;
1937
1938	/*
1939	* Allocate memory if we're supposed to do that.
1940	*/
1941	if (pVM->pgm.s.fRamPreAlloc)
1942	rc = pgmR3PhysRamPreAllocate(pVM);
1943
1944	LogRel(("PGMR3InitFinalize: 4 MB PSE mask %RGp\n", pVM->pgm.s.GCPhys4MBPSEMask));
1945	return rc;
1946	}
1947
1948
1949	/**
1950	* Applies relocations to data and code managed by this component.
1951	*
1952	* This function will be called at init and whenever the VMM need to relocate it
1953	* self inside the GC.
1954	*
1955	* @param pVM The VM.
1956	* @param offDelta Relocation delta relative to old location.
1957	*/
1958	VMMR3DECL(void) PGMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
1959	{
1960	LogFlow(("PGMR3Relocate %RGv to %RGv\n", pVM->pgm.s.GCPtrCR3Mapping, pVM->pgm.s.GCPtrCR3Mapping + offDelta));
1961
1962	/*
1963	* Paging stuff.
1964	*/
1965	pVM->pgm.s.GCPtrCR3Mapping += offDelta;
1966
1967	pgmR3ModeDataInit(pVM, true /* resolve GC/R0 symbols */);
1968
1969	/* Shadow, guest and both mode switch & relocation for each VCPU. */
1970	for (unsigned i=0;i<pVM->cCPUs;i++)
1971	{
1972	PVMCPU pVCpu = &pVM->aCpus[i];
1973
1974	pgmR3ModeDataSwitch(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
1975
1976	PGM_SHW_PFN(Relocate, pVCpu)(pVCpu, offDelta);
1977	PGM_GST_PFN(Relocate, pVCpu)(pVCpu, offDelta);
1978	PGM_BTH_PFN(Relocate, pVCpu)(pVCpu, offDelta);
1979	}
1980
1981	/*
1982	* Trees.
1983	*/
1984	pVM->pgm.s.pTreesRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pTreesR3);
1985
1986	/*
1987	* Ram ranges.
1988	*/
1989	if (pVM->pgm.s.pRamRangesR3)
1990	{
1991	/* Update the pSelfRC pointers and relink them. */
1992	for (PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
1993	if (!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING))
1994	pCur->pSelfRC = MMHyperCCToRC(pVM, pCur);
1995	pgmR3PhysRelinkRamRanges(pVM);
1996	}
1997
1998	/*
1999	* Update the two page directories with all page table mappings.
2000	* (One or more of them have changed, that's why we're here.)
2001	*/
2002	pVM->pgm.s.pMappingsRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pMappingsR3);
2003	for (PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3; pCur->pNextR3; pCur = pCur->pNextR3)
2004	pCur->pNextRC = MMHyperR3ToRC(pVM, pCur->pNextR3);
2005
2006	/* Relocate GC addresses of Page Tables. */
2007	for (PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3; pCur; pCur = pCur->pNextR3)
2008	{
2009	for (RTHCUINT i = 0; i < pCur->cPTs; i++)
2010	{
2011	pCur->aPTs[i].pPTRC = MMHyperR3ToRC(pVM, pCur->aPTs[i].pPTR3);
2012	pCur->aPTs[i].paPaePTsRC = MMHyperR3ToRC(pVM, pCur->aPTs[i].paPaePTsR3);
2013	}
2014	}
2015
2016	/*
2017	* Dynamic page mapping area.
2018	*/
2019	pVM->pgm.s.paDynPageMap32BitPTEsGC += offDelta;
2020	pVM->pgm.s.paDynPageMapPaePTEsGC += offDelta;
2021	pVM->pgm.s.pbDynPageMapBaseGC += offDelta;
2022
2023	/*
2024	* The Zero page.
2025	*/
2026	pVM->pgm.s.pvZeroPgR0 = MMHyperR3ToR0(pVM, pVM->pgm.s.pvZeroPgR3);
2027	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2028	AssertRelease(pVM->pgm.s.pvZeroPgR0 != NIL_RTR0PTR \|\| !VMMIsHwVirtExtForced(pVM));
2029	#else
2030	AssertRelease(pVM->pgm.s.pvZeroPgR0 != NIL_RTR0PTR);
2031	#endif
2032
2033	/*
2034	* Physical and virtual handlers.
2035	*/
2036	RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers, true, pgmR3RelocatePhysHandler, &offDelta);
2037	RTAvlroGCPtrDoWithAll(&pVM->pgm.s.pTreesR3->VirtHandlers, true, pgmR3RelocateVirtHandler, &offDelta);
2038	RTAvlroGCPtrDoWithAll(&pVM->pgm.s.pTreesR3->HyperVirtHandlers, true, pgmR3RelocateHyperVirtHandler, &offDelta);
2039
2040	/*
2041	* The page pool.
2042	*/
2043	pgmR3PoolRelocate(pVM);
2044	}
2045
2046
2047	/**
2048	* Callback function for relocating a physical access handler.
2049	*
2050	* @returns 0 (continue enum)
2051	* @param pNode Pointer to a PGMPHYSHANDLER node.
2052	* @param pvUser Pointer to the offDelta. This is a pointer to the delta since we're
2053	* not certain the delta will fit in a void pointer for all possible configs.
2054	*/
2055	static DECLCALLBACK(int) pgmR3RelocatePhysHandler(PAVLROGCPHYSNODECORE pNode, void *pvUser)
2056	{
2057	PPGMPHYSHANDLER pHandler = (PPGMPHYSHANDLER)pNode;
2058	RTGCINTPTR offDelta = *(PRTGCINTPTR)pvUser;
2059	if (pHandler->pfnHandlerRC)
2060	pHandler->pfnHandlerRC += offDelta;
2061	if (pHandler->pvUserRC >= 0x10000)
2062	pHandler->pvUserRC += offDelta;
2063	return 0;
2064	}
2065
2066
2067	/**
2068	* Callback function for relocating a virtual access handler.
2069	*
2070	* @returns 0 (continue enum)
2071	* @param pNode Pointer to a PGMVIRTHANDLER node.
2072	* @param pvUser Pointer to the offDelta. This is a pointer to the delta since we're
2073	* not certain the delta will fit in a void pointer for all possible configs.
2074	*/
2075	static DECLCALLBACK(int) pgmR3RelocateVirtHandler(PAVLROGCPTRNODECORE pNode, void *pvUser)
2076	{
2077	PPGMVIRTHANDLER pHandler = (PPGMVIRTHANDLER)pNode;
2078	RTGCINTPTR offDelta = *(PRTGCINTPTR)pvUser;
2079	Assert( pHandler->enmType == PGMVIRTHANDLERTYPE_ALL
2080	\|\| pHandler->enmType == PGMVIRTHANDLERTYPE_WRITE);
2081	Assert(pHandler->pfnHandlerRC);
2082	pHandler->pfnHandlerRC += offDelta;
2083	return 0;
2084	}
2085
2086
2087	/**
2088	* Callback function for relocating a virtual access handler for the hypervisor mapping.
2089	*
2090	* @returns 0 (continue enum)
2091	* @param pNode Pointer to a PGMVIRTHANDLER node.
2092	* @param pvUser Pointer to the offDelta. This is a pointer to the delta since we're
2093	* not certain the delta will fit in a void pointer for all possible configs.
2094	*/
2095	static DECLCALLBACK(int) pgmR3RelocateHyperVirtHandler(PAVLROGCPTRNODECORE pNode, void *pvUser)
2096	{
2097	PPGMVIRTHANDLER pHandler = (PPGMVIRTHANDLER)pNode;
2098	RTGCINTPTR offDelta = *(PRTGCINTPTR)pvUser;
2099	Assert(pHandler->enmType == PGMVIRTHANDLERTYPE_HYPERVISOR);
2100	Assert(pHandler->pfnHandlerRC);
2101	pHandler->pfnHandlerRC += offDelta;
2102	return 0;
2103	}
2104
2105
2106	/**
2107	* The VM is being reset.
2108	*
2109	* For the PGM component this means that any PD write monitors
2110	* needs to be removed.
2111	*
2112	* @param pVM VM handle.
2113	*/
2114	VMMR3DECL(void) PGMR3Reset(PVM pVM)
2115	{
2116	int rc;
2117
2118	LogFlow(("PGMR3Reset:\n"));
2119	VM_ASSERT_EMT(pVM);
2120
2121	pgmLock(pVM);
2122
2123	/*
2124	* Unfix any fixed mappings and disable CR3 monitoring.
2125	*/
2126	pVM->pgm.s.fMappingsFixed = false;
2127	pVM->pgm.s.GCPtrMappingFixed = 0;
2128	pVM->pgm.s.cbMappingFixed = 0;
2129
2130	/* Exit the guest paging mode before the pgm pool gets reset.
2131	* Important to clean up the amd64 case.
2132	*/
2133	for (unsigned i=0;i<pVM->cCPUs;i++)
2134	{
2135	PVMCPU pVCpu = &pVM->aCpus[i];
2136
2137	rc = PGM_GST_PFN(Exit, pVCpu)(pVCpu);
2138	AssertRC(rc);
2139	}
2140
2141	#ifdef DEBUG
2142	DBGFR3InfoLog(pVM, "mappings", NULL);
2143	DBGFR3InfoLog(pVM, "handlers", "all nostat");
2144	#endif
2145
2146	/*
2147	* Switch mode back to real mode. (before resetting the pgm pool!)
2148	*/
2149	for (unsigned i=0;i<pVM->cCPUs;i++)
2150	{
2151	PVMCPU pVCpu = &pVM->aCpus[i];
2152
2153	rc = PGMR3ChangeMode(pVM, pVCpu, PGMMODE_REAL);
2154	AssertRC(rc);
2155
2156	STAM_REL_COUNTER_RESET(&pVCpu->pgm.s.cGuestModeChanges);
2157	}
2158
2159	/*
2160	* Reset the shadow page pool.
2161	*/
2162	pgmR3PoolReset(pVM);
2163
2164	for (unsigned i=0;i<pVM->cCPUs;i++)
2165	{
2166	PVMCPU pVCpu = &pVM->aCpus[i];
2167
2168	/*
2169	* Re-init other members.
2170	*/
2171	pVCpu->pgm.s.fA20Enabled = true;
2172
2173	/*
2174	* Clear the FFs PGM owns.
2175	*/
2176	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2177	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2178	}
2179
2180	/*
2181	* Reset (zero) RAM pages.
2182	*/
2183	rc = pgmR3PhysRamReset(pVM);
2184	if (RT_SUCCESS(rc))
2185	{
2186	/*
2187	* Reset (zero) shadow ROM pages.
2188	*/
2189	rc = pgmR3PhysRomReset(pVM);
2190	}
2191
2192	pgmUnlock(pVM);
2193	//return rc;
2194	AssertReleaseRC(rc);
2195	}
2196
2197
2198	#ifdef VBOX_STRICT
2199	/**
2200	* VM state change callback for clearing fNoMorePhysWrites after
2201	* a snapshot has been created.
2202	*/
2203	static DECLCALLBACK(void) pgmR3ResetNoMorePhysWritesFlag(PVM pVM, VMSTATE enmState, VMSTATE enmOldState, void *pvUser)
2204	{
2205	if (enmState == VMSTATE_RUNNING)
2206	pVM->pgm.s.fNoMorePhysWrites = false;
2207	}
2208	#endif
2209
2210
2211	/**
2212	* Terminates the PGM.
2213	*
2214	* @returns VBox status code.
2215	* @param pVM Pointer to VM structure.
2216	*/
2217	VMMR3DECL(int) PGMR3Term(PVM pVM)
2218	{
2219	PGMDeregisterStringFormatTypes();
2220	return PDMR3CritSectDelete(&pVM->pgm.s.CritSect);
2221	}
2222
2223
2224	/**
2225	* Terminates the per-VCPU PGM.
2226	*
2227	* Termination means cleaning up and freeing all resources,
2228	* the VM it self is at this point powered off or suspended.
2229	*
2230	* @returns VBox status code.
2231	* @param pVM The VM to operate on.
2232	*/
2233	VMMR3DECL(int) PGMR3TermCPU(PVM pVM)
2234	{
2235	return 0;
2236	}
2237
2238
2239	/**
2240	* Find the ROM tracking structure for the given page.
2241	*
2242	* @returns Pointer to the ROM page structure. NULL if the caller didn't check
2243	* that it's a ROM page.
2244	* @param pVM The VM handle.
2245	* @param GCPhys The address of the ROM page.
2246	*/
2247	static PPGMROMPAGE pgmR3GetRomPage(PVM pVM, RTGCPHYS GCPhys)
2248	{
2249	for (PPGMROMRANGE pRomRange = pVM->pgm.s.CTX_SUFF(pRomRanges);
2250	pRomRange;
2251	pRomRange = pRomRange->CTX_SUFF(pNext))
2252	{
2253	RTGCPHYS off = GCPhys - pRomRange->GCPhys;
2254	if (GCPhys - pRomRange->GCPhys < pRomRange->cb)
2255	return &pRomRange->aPages[off >> PAGE_SHIFT];
2256	}
2257	return NULL;
2258	}
2259
2260
2261	/**
2262	* Save zero indicator + bits for the specified page.
2263	*
2264	* @returns VBox status code, errors are logged/asserted before returning.
2265	* @param pVM The VM handle.
2266	* @param pSSH The saved state handle.
2267	* @param pPage The page to save.
2268	* @param GCPhys The address of the page.
2269	* @param pRam The ram range (for error logging).
2270	*/
2271	static int pgmR3SavePage(PVM pVM, PSSMHANDLE pSSM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
2272	{
2273	int rc;
2274	if (PGM_PAGE_IS_ZERO(pPage))
2275	rc = SSMR3PutU8(pSSM, 0);
2276	else
2277	{
2278	void const *pvPage;
2279	rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvPage);
2280	AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] GCPhys=%#x %s\n", pPage, GCPhys, pRam->pszDesc), rc);
2281
2282	SSMR3PutU8(pSSM, 1);
2283	rc = SSMR3PutMem(pSSM, pvPage, PAGE_SIZE);
2284	}
2285	return rc;
2286	}
2287
2288
2289	/**
2290	* Save a shadowed ROM page.
2291	*
2292	* Format: Type, protection, and two pages with zero indicators.
2293	*
2294	* @returns VBox status code, errors are logged/asserted before returning.
2295	* @param pVM The VM handle.
2296	* @param pSSH The saved state handle.
2297	* @param pPage The page to save.
2298	* @param GCPhys The address of the page.
2299	* @param pRam The ram range (for error logging).
2300	*/
2301	static int pgmR3SaveShadowedRomPage(PVM pVM, PSSMHANDLE pSSM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
2302	{
2303	/* Need to save both pages and the current state. */
2304	PPGMROMPAGE pRomPage = pgmR3GetRomPage(pVM, GCPhys);
2305	AssertLogRelMsgReturn(pRomPage, ("GCPhys=%RGp %s\n", GCPhys, pRam->pszDesc), VERR_INTERNAL_ERROR);
2306
2307	SSMR3PutU8(pSSM, PGMPAGETYPE_ROM_SHADOW);
2308	SSMR3PutU8(pSSM, pRomPage->enmProt);
2309
2310	int rc = pgmR3SavePage(pVM, pSSM, pPage, GCPhys, pRam);
2311	if (RT_SUCCESS(rc))
2312	{
2313	PPGMPAGE pPagePassive = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
2314	rc = pgmR3SavePage(pVM, pSSM, pPagePassive, GCPhys, pRam);
2315	}
2316	return rc;
2317	}
2318
2319	/** PGM fields to save/load. */
2320	static const SSMFIELD s_aPGMFields[] =
2321	{
2322	SSMFIELD_ENTRY( PGM, fMappingsFixed),
2323	SSMFIELD_ENTRY_GCPTR( PGM, GCPtrMappingFixed),
2324	SSMFIELD_ENTRY( PGM, cbMappingFixed),
2325	SSMFIELD_ENTRY_TERM()
2326	};
2327
2328	static const SSMFIELD s_aPGMCpuFields[] =
2329	{
2330	SSMFIELD_ENTRY( PGMCPU, fA20Enabled),
2331	SSMFIELD_ENTRY_GCPHYS( PGMCPU, GCPhysA20Mask),
2332	SSMFIELD_ENTRY( PGMCPU, enmGuestMode),
2333	SSMFIELD_ENTRY_TERM()
2334	};
2335
2336	/* For loading old saved states. (pre-smp) */
2337	typedef struct
2338	{
2339	/** If set no conflict checks are required. (boolean) */
2340	bool fMappingsFixed;
2341	/** Size of fixed mapping */
2342	uint32_t cbMappingFixed;
2343	/** Base address (GC) of fixed mapping */
2344	RTGCPTR GCPtrMappingFixed;
2345	/** A20 gate mask.
2346	* Our current approach to A20 emulation is to let REM do it and don't bother
2347	* anywhere else. The interesting Guests will be operating with it enabled anyway.
2348	* But whould need arrise, we'll subject physical addresses to this mask. */
2349	RTGCPHYS GCPhysA20Mask;
2350	/** A20 gate state - boolean! */
2351	bool fA20Enabled;
2352	/** The guest paging mode. */
2353	PGMMODE enmGuestMode;
2354	} PGMOLD;
2355
2356	static const SSMFIELD s_aPGMFields_Old[] =
2357	{
2358	SSMFIELD_ENTRY( PGMOLD, fMappingsFixed),
2359	SSMFIELD_ENTRY_GCPTR( PGMOLD, GCPtrMappingFixed),
2360	SSMFIELD_ENTRY( PGMOLD, cbMappingFixed),
2361	SSMFIELD_ENTRY( PGMOLD, fA20Enabled),
2362	SSMFIELD_ENTRY_GCPHYS( PGMOLD, GCPhysA20Mask),
2363	SSMFIELD_ENTRY( PGMOLD, enmGuestMode),
2364	SSMFIELD_ENTRY_TERM()
2365	};
2366
2367
2368	/**
2369	* Execute state save operation.
2370	*
2371	* @returns VBox status code.
2372	* @param pVM VM Handle.
2373	* @param pSSM SSM operation handle.
2374	*/
2375	static DECLCALLBACK(int) pgmR3Save(PVM pVM, PSSMHANDLE pSSM)
2376	{
2377	int rc;
2378	unsigned i;
2379	PPGM pPGM = &pVM->pgm.s;
2380
2381	/*
2382	* Lock PGM and set the no-more-writes indicator.
2383	*/
2384	pgmLock(pVM);
2385	pVM->pgm.s.fNoMorePhysWrites = true;
2386
2387	/*
2388	* Save basic data (required / unaffected by relocation).
2389	*/
2390	SSMR3PutStruct(pSSM, pPGM, &s_aPGMFields[0]);
2391
2392	for (i=0;i<pVM->cCPUs;i++)
2393	{
2394	PVMCPU pVCpu = &pVM->aCpus[i];
2395
2396	SSMR3PutStruct(pSSM, &pVCpu->pgm.s, &s_aPGMCpuFields[0]);
2397	}
2398
2399	/*
2400	* The guest mappings.
2401	*/
2402	i = 0;
2403	for (PPGMMAPPING pMapping = pPGM->pMappingsR3; pMapping; pMapping = pMapping->pNextR3, i++)
2404	{
2405	SSMR3PutU32( pSSM, i);
2406	SSMR3PutStrZ( pSSM, pMapping->pszDesc); /* This is the best unique id we have... */
2407	SSMR3PutGCPtr( pSSM, pMapping->GCPtr);
2408	SSMR3PutGCUIntPtr(pSSM, pMapping->cPTs);
2409	}
2410	rc = SSMR3PutU32(pSSM, ~0); /* terminator. */
2411
2412	/*
2413	* Ram ranges and the memory they describe.
2414	*/
2415	i = 0;
2416	for (PPGMRAMRANGE pRam = pPGM->pRamRangesR3; pRam; pRam = pRam->pNextR3, i++)
2417	{
2418	/*
2419	* Save the ram range details.
2420	*/
2421	SSMR3PutU32(pSSM, i);
2422	SSMR3PutGCPhys(pSSM, pRam->GCPhys);
2423	SSMR3PutGCPhys(pSSM, pRam->GCPhysLast);
2424	SSMR3PutGCPhys(pSSM, pRam->cb);
2425	SSMR3PutU8(pSSM, !!pRam->pvR3); /* Boolean indicating memory or not. */
2426	SSMR3PutStrZ(pSSM, pRam->pszDesc); /* This is the best unique id we have... */
2427
2428	/*
2429	* Iterate the pages, only two special case.
2430	*/
2431	uint32_t const cPages = pRam->cb >> PAGE_SHIFT;
2432	for (uint32_t iPage = 0; iPage < cPages; iPage++)
2433	{
2434	RTGCPHYS GCPhysPage = pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
2435	PPGMPAGE pPage = &pRam->aPages[iPage];
2436	uint8_t uType = PGM_PAGE_GET_TYPE(pPage);
2437
2438	if (uType == PGMPAGETYPE_ROM_SHADOW)
2439	rc = pgmR3SaveShadowedRomPage(pVM, pSSM, pPage, GCPhysPage, pRam);
2440	else if (uType == PGMPAGETYPE_MMIO2_ALIAS_MMIO)
2441	{
2442	/* MMIO2 alias -> MMIO; the device will just have to deal with this. */
2443	SSMR3PutU8(pSSM, PGMPAGETYPE_MMIO);
2444	rc = SSMR3PutU8(pSSM, 0 /* ZERO */);
2445	}
2446	else
2447	{
2448	SSMR3PutU8(pSSM, uType);
2449	rc = pgmR3SavePage(pVM, pSSM, pPage, GCPhysPage, pRam);
2450	}
2451	if (RT_FAILURE(rc))
2452	break;
2453	}
2454	if (RT_FAILURE(rc))
2455	break;
2456	}
2457
2458	pgmUnlock(pVM);
2459	return SSMR3PutU32(pSSM, ~0); /* terminator. */
2460	}
2461
2462
2463	/**
2464	* Load an ignored page.
2465	*
2466	* @returns VBox status code.
2467	* @param pSSM The saved state handle.
2468	*/
2469	static int pgmR3LoadPageToDevNull(PSSMHANDLE pSSM)
2470	{
2471	uint8_t abPage[PAGE_SIZE];
2472	return SSMR3GetMem(pSSM, &abPage[0], sizeof(abPage));
2473	}
2474
2475
2476	/**
2477	* Loads a page without any bits in the saved state, i.e. making sure it's
2478	* really zero.
2479	*
2480	* @returns VBox status code.
2481	* @param pVM The VM handle.
2482	* @param uType The page type or PGMPAGETYPE_INVALID (old saved
2483	* state).
2484	* @param pPage The guest page tracking structure.
2485	* @param GCPhys The page address.
2486	* @param pRam The ram range (logging).
2487	*/
2488	static int pgmR3LoadPageZero(PVM pVM, uint8_t uType, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
2489	{
2490	if ( PGM_PAGE_GET_TYPE(pPage) != uType
2491	&& uType != PGMPAGETYPE_INVALID)
2492	return VERR_SSM_UNEXPECTED_DATA;
2493
2494	/* I think this should be sufficient. */
2495	if (!PGM_PAGE_IS_ZERO(pPage))
2496	return VERR_SSM_UNEXPECTED_DATA;
2497
2498	NOREF(pVM);
2499	NOREF(GCPhys);
2500	NOREF(pRam);
2501	return VINF_SUCCESS;
2502	}
2503
2504
2505	/**
2506	* Loads a page from the saved state.
2507	*
2508	* @returns VBox status code.
2509	* @param pVM The VM handle.
2510	* @param pSSM The SSM handle.
2511	* @param uType The page type or PGMPAGETYEP_INVALID (old saved
2512	* state).
2513	* @param pPage The guest page tracking structure.
2514	* @param GCPhys The page address.
2515	* @param pRam The ram range (logging).
2516	*/
2517	static int pgmR3LoadPageBits(PVM pVM, PSSMHANDLE pSSM, uint8_t uType, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
2518	{
2519	int rc;
2520
2521	/*
2522	* Match up the type, dealing with MMIO2 aliases (dropped).
2523	*/
2524	AssertLogRelMsgReturn( PGM_PAGE_GET_TYPE(pPage) == uType
2525	\|\| uType == PGMPAGETYPE_INVALID,
2526	("pPage=%R[pgmpage] GCPhys=%#x %s\n", pPage, GCPhys, pRam->pszDesc),
2527	VERR_SSM_UNEXPECTED_DATA);
2528
2529	/*
2530	* Load the page.
2531	*/
2532	void *pvPage;
2533	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvPage);
2534	if (RT_SUCCESS(rc))
2535	rc = SSMR3GetMem(pSSM, pvPage, PAGE_SIZE);
2536
2537	return rc;
2538	}
2539
2540
2541	/**
2542	* Loads a page (counter part to pgmR3SavePage).
2543	*
2544	* @returns VBox status code, fully bitched errors.
2545	* @param pVM The VM handle.
2546	* @param pSSM The SSM handle.
2547	* @param uType The page type.
2548	* @param pPage The page.
2549	* @param GCPhys The page address.
2550	* @param pRam The RAM range (for error messages).
2551	*/
2552	static int pgmR3LoadPage(PVM pVM, PSSMHANDLE pSSM, uint8_t uType, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
2553	{
2554	uint8_t uState;
2555	int rc = SSMR3GetU8(pSSM, &uState);
2556	AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] GCPhys=%#x %s rc=%Rrc\n", pPage, GCPhys, pRam->pszDesc, rc), rc);
2557	if (uState == 0 /* zero */)
2558	rc = pgmR3LoadPageZero(pVM, uType, pPage, GCPhys, pRam);
2559	else if (uState == 1)
2560	rc = pgmR3LoadPageBits(pVM, pSSM, uType, pPage, GCPhys, pRam);
2561	else
2562	rc = VERR_INTERNAL_ERROR;
2563	AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] uState=%d uType=%d GCPhys=%RGp %s rc=%Rrc\n",
2564	pPage, uState, uType, GCPhys, pRam->pszDesc, rc),
2565	rc);
2566	return VINF_SUCCESS;
2567	}
2568
2569
2570	/**
2571	* Loads a shadowed ROM page.
2572	*
2573	* @returns VBox status code, errors are fully bitched.
2574	* @param pVM The VM handle.
2575	* @param pSSM The saved state handle.
2576	* @param pPage The page.
2577	* @param GCPhys The page address.
2578	* @param pRam The RAM range (for error messages).
2579	*/
2580	static int pgmR3LoadShadowedRomPage(PVM pVM, PSSMHANDLE pSSM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
2581	{
2582	/*
2583	* Load and set the protection first, then load the two pages, the first
2584	* one is the active the other is the passive.
2585	*/
2586	PPGMROMPAGE pRomPage = pgmR3GetRomPage(pVM, GCPhys);
2587	AssertLogRelMsgReturn(pRomPage, ("GCPhys=%RGp %s\n", GCPhys, pRam->pszDesc), VERR_INTERNAL_ERROR);
2588
2589	uint8_t uProt;
2590	int rc = SSMR3GetU8(pSSM, &uProt);
2591	AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] GCPhys=%#x %s\n", pPage, GCPhys, pRam->pszDesc), rc);
2592	PGMROMPROT enmProt = (PGMROMPROT)uProt;
2593	AssertLogRelMsgReturn( enmProt >= PGMROMPROT_INVALID
2594	&& enmProt < PGMROMPROT_END,
2595	("enmProt=%d pPage=%R[pgmpage] GCPhys=%#x %s\n", enmProt, pPage, GCPhys, pRam->pszDesc),
2596	VERR_SSM_UNEXPECTED_DATA);
2597
2598	if (pRomPage->enmProt != enmProt)
2599	{
2600	rc = PGMR3PhysRomProtect(pVM, GCPhys, PAGE_SIZE, enmProt);
2601	AssertLogRelRCReturn(rc, rc);
2602	AssertLogRelReturn(pRomPage->enmProt == enmProt, VERR_INTERNAL_ERROR);
2603	}
2604
2605	PPGMPAGE pPageActive = PGMROMPROT_IS_ROM(enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
2606	PPGMPAGE pPagePassive = PGMROMPROT_IS_ROM(enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
2607	uint8_t u8ActiveType = PGMROMPROT_IS_ROM(enmProt) ? PGMPAGETYPE_ROM : PGMPAGETYPE_ROM_SHADOW;
2608	uint8_t u8PassiveType= PGMROMPROT_IS_ROM(enmProt) ? PGMPAGETYPE_ROM_SHADOW : PGMPAGETYPE_ROM;
2609
2610	rc = pgmR3LoadPage(pVM, pSSM, u8ActiveType, pPage, GCPhys, pRam);
2611	if (RT_SUCCESS(rc))
2612	{
2613	pPageActive = pPage;
2614	rc = pgmR3LoadPage(pVM, pSSM, u8PassiveType, pPagePassive, GCPhys, pRam);
2615	}
2616	return rc;
2617	}
2618
2619
2620	/**
2621	* Worker for pgmR3Load.
2622	*
2623	* @returns VBox status code.
2624	*
2625	* @param pVM The VM handle.
2626	* @param pSSM The SSM handle.
2627	* @param u32Version The saved state version.
2628	*/
2629	static int pgmR3LoadLocked(PVM pVM, PSSMHANDLE pSSM, uint32_t u32Version)
2630	{
2631	int rc;
2632	PPGM pPGM = &pVM->pgm.s;
2633	uint32_t u32Sep;
2634
2635	/*
2636	* Load basic data (required / unaffected by relocation).
2637	*/
2638	if (u32Version >= PGM_SAVED_STATE_VERSION)
2639	{
2640	rc = SSMR3GetStruct(pSSM, pPGM, &s_aPGMFields[0]);
2641	AssertLogRelRCReturn(rc, rc);
2642
2643	for (unsigned i=0;i<pVM->cCPUs;i++)
2644	{
2645	PVMCPU pVCpu = &pVM->aCpus[i];
2646
2647	rc = SSMR3GetStruct(pSSM, &pVCpu->pgm.s, &s_aPGMCpuFields[0]);
2648	AssertLogRelRCReturn(rc, rc);
2649	}
2650	}
2651	else
2652	if (u32Version >= PGM_SAVED_STATE_VERSION_RR_DESC)
2653	{
2654	PGMOLD pgmOld;
2655
2656	AssertRelease(pVM->cCPUs == 1);
2657
2658	rc = SSMR3GetStruct(pSSM, &pgmOld, &s_aPGMFields_Old[0]);
2659	AssertLogRelRCReturn(rc, rc);
2660
2661	pPGM->fMappingsFixed = pgmOld.fMappingsFixed;
2662	pPGM->GCPtrMappingFixed = pgmOld.GCPtrMappingFixed;
2663	pPGM->cbMappingFixed = pgmOld.cbMappingFixed;
2664
2665	pVM->aCpus[0].pgm.s.fA20Enabled = pgmOld.fA20Enabled;
2666	pVM->aCpus[0].pgm.s.GCPhysA20Mask = pgmOld.GCPhysA20Mask;
2667	pVM->aCpus[0].pgm.s.enmGuestMode = pgmOld.enmGuestMode;
2668	}
2669	else
2670	{
2671	AssertRelease(pVM->cCPUs == 1);
2672
2673	SSMR3GetBool(pSSM, &pPGM->fMappingsFixed);
2674	SSMR3GetGCPtr(pSSM, &pPGM->GCPtrMappingFixed);
2675	SSMR3GetU32(pSSM, &pPGM->cbMappingFixed);
2676
2677	uint32_t cbRamSizeIgnored;
2678	rc = SSMR3GetU32(pSSM, &cbRamSizeIgnored);
2679	if (RT_FAILURE(rc))
2680	return rc;
2681	SSMR3GetGCPhys(pSSM, &pVM->aCpus[0].pgm.s.GCPhysA20Mask);
2682
2683	uint32_t u32 = 0;
2684	SSMR3GetUInt(pSSM, &u32);
2685	pVM->aCpus[0].pgm.s.fA20Enabled = !!u32;
2686	SSMR3GetUInt(pSSM, &pVM->aCpus[0].pgm.s.fSyncFlags);
2687	RTUINT uGuestMode;
2688	SSMR3GetUInt(pSSM, &uGuestMode);
2689	pVM->aCpus[0].pgm.s.enmGuestMode = (PGMMODE)uGuestMode;
2690
2691	/* check separator. */
2692	SSMR3GetU32(pSSM, &u32Sep);
2693	if (RT_FAILURE(rc))
2694	return rc;
2695	if (u32Sep != (uint32_t)~0)
2696	{
2697	AssertMsgFailed(("u32Sep=%#x (first)\n", u32Sep));
2698	return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
2699	}
2700	}
2701
2702	/*
2703	* The guest mappings.
2704	*/
2705	uint32_t i = 0;
2706	for (;; i++)
2707	{
2708	/* Check the seqence number / separator. */
2709	rc = SSMR3GetU32(pSSM, &u32Sep);
2710	if (RT_FAILURE(rc))
2711	return rc;
2712	if (u32Sep == ~0U)
2713	break;
2714	if (u32Sep != i)
2715	{
2716	AssertMsgFailed(("u32Sep=%#x (last)\n", u32Sep));
2717	return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
2718	}
2719
2720	/* get the mapping details. */
2721	char szDesc[256];
2722	szDesc[0] = '\0';
2723	rc = SSMR3GetStrZ(pSSM, szDesc, sizeof(szDesc));
2724	if (RT_FAILURE(rc))
2725	return rc;
2726	RTGCPTR GCPtr;
2727	SSMR3GetGCPtr(pSSM, &GCPtr);
2728	RTGCPTR cPTs;
2729	rc = SSMR3GetGCUIntPtr(pSSM, &cPTs);
2730	if (RT_FAILURE(rc))
2731	return rc;
2732
2733	/* find matching range. */
2734	PPGMMAPPING pMapping;
2735	for (pMapping = pPGM->pMappingsR3; pMapping; pMapping = pMapping->pNextR3)
2736	if ( pMapping->cPTs == cPTs
2737	&& !strcmp(pMapping->pszDesc, szDesc))
2738	break;
2739	AssertLogRelMsgReturn(pMapping, ("Couldn't find mapping: cPTs=%#x szDesc=%s (GCPtr=%RGv)\n",
2740	cPTs, szDesc, GCPtr),
2741	VERR_SSM_LOAD_CONFIG_MISMATCH);
2742
2743	/* relocate it. */
2744	if (pMapping->GCPtr != GCPtr)
2745	{
2746	AssertMsg((GCPtr >> X86_PD_SHIFT << X86_PD_SHIFT) == GCPtr, ("GCPtr=%RGv\n", GCPtr));
2747	pgmR3MapRelocate(pVM, pMapping, pMapping->GCPtr, GCPtr);
2748	}
2749	else
2750	Log(("pgmR3Load: '%s' needed no relocation (%RGv)\n", szDesc, GCPtr));
2751	}
2752
2753	/*
2754	* Ram range flags and bits.
2755	*/
2756	i = 0;
2757	for (PPGMRAMRANGE pRam = pPGM->pRamRangesR3; pRam; pRam = pRam->pNextR3, i++)
2758	{
2759	/** @todo MMIO ranges may move (PCI reconfig), we currently assume they don't. */
2760
2761	/* Check the seqence number / separator. */
2762	rc = SSMR3GetU32(pSSM, &u32Sep);
2763	if (RT_FAILURE(rc))
2764	return rc;
2765	if (u32Sep == ~0U)
2766	break;
2767	if (u32Sep != i)
2768	{
2769	AssertMsgFailed(("u32Sep=%#x (last)\n", u32Sep));
2770	return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
2771	}
2772
2773	/* Get the range details. */
2774	RTGCPHYS GCPhys;
2775	SSMR3GetGCPhys(pSSM, &GCPhys);
2776	RTGCPHYS GCPhysLast;
2777	SSMR3GetGCPhys(pSSM, &GCPhysLast);
2778	RTGCPHYS cb;
2779	SSMR3GetGCPhys(pSSM, &cb);
2780	uint8_t fHaveBits;
2781	rc = SSMR3GetU8(pSSM, &fHaveBits);
2782	if (RT_FAILURE(rc))
2783	return rc;
2784	if (fHaveBits & ~1)
2785	{
2786	AssertMsgFailed(("u32Sep=%#x (last)\n", u32Sep));
2787	return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
2788	}
2789	size_t cchDesc = 0;
2790	char szDesc[256];
2791	szDesc[0] = '\0';
2792	if (u32Version >= PGM_SAVED_STATE_VERSION_RR_DESC)
2793	{
2794	rc = SSMR3GetStrZ(pSSM, szDesc, sizeof(szDesc));
2795	if (RT_FAILURE(rc))
2796	return rc;
2797	/* Since we've modified the description strings in r45878, only compare
2798	them if the saved state is more recent. */
2799	if (u32Version != PGM_SAVED_STATE_VERSION_RR_DESC)
2800	cchDesc = strlen(szDesc);
2801	}
2802
2803	/*
2804	* Match it up with the current range.
2805	*
2806	* Note there is a hack for dealing with the high BIOS mapping
2807	* in the old saved state format, this means we might not have
2808	* a 1:1 match on success.
2809	*/
2810	if ( ( GCPhys != pRam->GCPhys
2811	\|\| GCPhysLast != pRam->GCPhysLast
2812	\|\| cb != pRam->cb
2813	\|\| ( cchDesc
2814	&& strcmp(szDesc, pRam->pszDesc)) )
2815	/* Hack for PDMDevHlpPhysReserve(pDevIns, 0xfff80000, 0x80000, "High ROM Region"); */
2816	&& ( u32Version != PGM_SAVED_STATE_VERSION_OLD_PHYS_CODE
2817	\|\| GCPhys != UINT32_C(0xfff80000)
2818	\|\| GCPhysLast != UINT32_C(0xffffffff)
2819	\|\| pRam->GCPhysLast != GCPhysLast
2820	\|\| pRam->GCPhys < GCPhys
2821	\|\| !fHaveBits)
2822	)
2823	{
2824	LogRel(("Ram range: %RGp-%RGp %RGp bytes %s %s\n"
2825	"State : %RGp-%RGp %RGp bytes %s %s\n",
2826	pRam->GCPhys, pRam->GCPhysLast, pRam->cb, pRam->pvR3 ? "bits" : "nobits", pRam->pszDesc,
2827	GCPhys, GCPhysLast, cb, fHaveBits ? "bits" : "nobits", szDesc));
2828	/*
2829	* If we're loading a state for debugging purpose, don't make a fuss if
2830	* the MMIO and ROM stuff isn't 100% right, just skip the mismatches.
2831	*/
2832	if ( SSMR3HandleGetAfter(pSSM) != SSMAFTER_DEBUG_IT
2833	\|\| GCPhys < 8 * _1M)
2834	AssertFailedReturn(VERR_SSM_LOAD_CONFIG_MISMATCH);
2835
2836	AssertMsgFailed(("debug skipping not implemented, sorry\n"));
2837	continue;
2838	}
2839
2840	uint32_t cPages = (GCPhysLast - GCPhys + 1) >> PAGE_SHIFT;
2841	if (u32Version >= PGM_SAVED_STATE_VERSION_RR_DESC)
2842	{
2843	/*
2844	* Load the pages one by one.
2845	*/
2846	for (uint32_t iPage = 0; iPage < cPages; iPage++)
2847	{
2848	RTGCPHYS const GCPhysPage = ((RTGCPHYS)iPage << PAGE_SHIFT) + pRam->GCPhys;
2849	PPGMPAGE pPage = &pRam->aPages[iPage];
2850	uint8_t uType;
2851	rc = SSMR3GetU8(pSSM, &uType);
2852	AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] iPage=%#x GCPhysPage=%#x %s\n", pPage, iPage, GCPhysPage, pRam->pszDesc), rc);
2853	if (uType == PGMPAGETYPE_ROM_SHADOW)
2854	rc = pgmR3LoadShadowedRomPage(pVM, pSSM, pPage, GCPhysPage, pRam);
2855	else
2856	rc = pgmR3LoadPage(pVM, pSSM, uType, pPage, GCPhysPage, pRam);
2857	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhysPage=%#x %s\n", rc, iPage, GCPhysPage, pRam->pszDesc), rc);
2858	}
2859	}
2860	else
2861	{
2862	/*
2863	* Old format.
2864	*/
2865	AssertLogRelReturn(!pVM->pgm.s.fRamPreAlloc, VERR_NOT_SUPPORTED); /* can't be detected. */
2866
2867	/* Of the page flags, pick up MMIO2 and ROM/RESERVED for the !fHaveBits case.
2868	The rest is generally irrelevant and wrong since the stuff have to match registrations. */
2869	uint32_t fFlags = 0;
2870	for (uint32_t iPage = 0; iPage < cPages; iPage++)
2871	{
2872	uint16_t u16Flags;
2873	rc = SSMR3GetU16(pSSM, &u16Flags);
2874	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhys=%#x %s\n", rc, iPage, pRam->GCPhys, pRam->pszDesc), rc);
2875	fFlags \|= u16Flags;
2876	}
2877
2878	/* Load the bits */
2879	if ( !fHaveBits
2880	&& GCPhysLast < UINT32_C(0xe0000000))
2881	{
2882	/*
2883	* Dynamic chunks.
2884	*/
2885	const uint32_t cPagesInChunk = (110241024) >> PAGE_SHIFT;
2886	AssertLogRelMsgReturn(cPages % cPagesInChunk == 0,
2887	("cPages=%#x cPagesInChunk=%#x\n", cPages, cPagesInChunk, pRam->GCPhys, pRam->pszDesc),
2888	VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
2889
2890	for (uint32_t iPage = 0; iPage < cPages; /* incremented by inner loop */ )
2891	{
2892	uint8_t fPresent;
2893	rc = SSMR3GetU8(pSSM, &fPresent);
2894	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhys=%#x %s\n", rc, iPage, pRam->GCPhys, pRam->pszDesc), rc);
2895	AssertLogRelMsgReturn(fPresent == (uint8_t)true \|\| fPresent == (uint8_t)false,
2896	("fPresent=%#x iPage=%#x GCPhys=%#x %s\n", fPresent, iPage, pRam->GCPhys, pRam->pszDesc),
2897	VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
2898
2899	for (uint32_t iChunkPage = 0; iChunkPage < cPagesInChunk; iChunkPage++, iPage++)
2900	{
2901	RTGCPHYS const GCPhysPage = ((RTGCPHYS)iPage << PAGE_SHIFT) + pRam->GCPhys;
2902	PPGMPAGE pPage = &pRam->aPages[iPage];
2903	if (fPresent)
2904	{
2905	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO)
2906	rc = pgmR3LoadPageToDevNull(pSSM);
2907	else
2908	rc = pgmR3LoadPageBits(pVM, pSSM, PGMPAGETYPE_INVALID, pPage, GCPhysPage, pRam);
2909	}
2910	else
2911	rc = pgmR3LoadPageZero(pVM, PGMPAGETYPE_INVALID, pPage, GCPhysPage, pRam);
2912	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhysPage=%#x %s\n", rc, iPage, GCPhysPage, pRam->pszDesc), rc);
2913	}
2914	}
2915	}
2916	else if (pRam->pvR3)
2917	{
2918	/*
2919	* MMIO2.
2920	*/
2921	AssertLogRelMsgReturn((fFlags & 0x0f) == RT_BIT(3) /MM_RAM_FLAGS_MMIO2/,
2922	("fFlags=%#x GCPhys=%#x %s\n", fFlags, pRam->GCPhys, pRam->pszDesc),
2923	VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
2924	AssertLogRelMsgReturn(pRam->pvR3,
2925	("GCPhys=%#x %s\n", pRam->GCPhys, pRam->pszDesc),
2926	VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
2927
2928	rc = SSMR3GetMem(pSSM, pRam->pvR3, pRam->cb);
2929	AssertLogRelMsgRCReturn(rc, ("GCPhys=%#x %s\n", pRam->GCPhys, pRam->pszDesc), rc);
2930	}
2931	else if (GCPhysLast < UINT32_C(0xfff80000))
2932	{
2933	/*
2934	* PCI MMIO, no pages saved.
2935	*/
2936	}
2937	else
2938	{
2939	/*
2940	* Load the 0xfff80000..0xffffffff BIOS range.
2941	* It starts with X reserved pages that we have to skip over since
2942	* the RAMRANGE create by the new code won't include those.
2943	*/
2944	AssertLogRelMsgReturn( !(fFlags & RT_BIT(3) /MM_RAM_FLAGS_MMIO2/)
2945	&& (fFlags & RT_BIT(0) /MM_RAM_FLAGS_RESERVED/),
2946	("fFlags=%#x GCPhys=%#x %s\n", fFlags, pRam->GCPhys, pRam->pszDesc),
2947	VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
2948	AssertLogRelMsgReturn(GCPhys == UINT32_C(0xfff80000),
2949	("GCPhys=%RGp pRamRange{GCPhys=%#x %s}\n", GCPhys, pRam->GCPhys, pRam->pszDesc),
2950	VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
2951
2952	/* Skip wasted reserved pages before the ROM. */
2953	while (GCPhys < pRam->GCPhys)
2954	{
2955	rc = pgmR3LoadPageToDevNull(pSSM);
2956	GCPhys += PAGE_SIZE;
2957	}
2958
2959	/* Load the bios pages. */
2960	cPages = pRam->cb >> PAGE_SHIFT;
2961	for (uint32_t iPage = 0; iPage < cPages; iPage++)
2962	{
2963	RTGCPHYS const GCPhysPage = ((RTGCPHYS)iPage << PAGE_SHIFT) + pRam->GCPhys;
2964	PPGMPAGE pPage = &pRam->aPages[iPage];
2965
2966	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_ROM,
2967	("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, GCPhys),
2968	VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
2969	rc = pgmR3LoadPageBits(pVM, pSSM, PGMPAGETYPE_ROM, pPage, GCPhysPage, pRam);
2970	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhys=%#x %s\n", rc, iPage, pRam->GCPhys, pRam->pszDesc), rc);
2971	}
2972	}
2973	}
2974	}
2975
2976	return rc;
2977	}
2978
2979
2980	/**
2981	* Execute state load operation.
2982	*
2983	* @returns VBox status code.
2984	* @param pVM VM Handle.
2985	* @param pSSM SSM operation handle.
2986	* @param u32Version Data layout version.
2987	*/
2988	static DECLCALLBACK(int) pgmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t u32Version)
2989	{
2990	int rc;
2991	PPGM pPGM = &pVM->pgm.s;
2992
2993	/*
2994	* Validate version.
2995	*/
2996	if ( u32Version != PGM_SAVED_STATE_VERSION
2997	&& u32Version != PGM_SAVED_STATE_VERSION_2_2_2
2998	&& u32Version != PGM_SAVED_STATE_VERSION_RR_DESC
2999	&& u32Version != PGM_SAVED_STATE_VERSION_OLD_PHYS_CODE)
3000	{
3001	AssertMsgFailed(("pgmR3Load: Invalid version u32Version=%d (current %d)!\n", u32Version, PGM_SAVED_STATE_VERSION));
3002	return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
3003	}
3004
3005	/*
3006	* Call the reset function to make sure all the memory is cleared.
3007	*/
3008	PGMR3Reset(pVM);
3009
3010	/*
3011	* Do the loading while owning the lock because a bunch of the functions
3012	* we're using requires this.
3013	*/
3014	pgmLock(pVM);
3015	rc = pgmR3LoadLocked(pVM, pSSM, u32Version);
3016	pgmUnlock(pVM);
3017	if (RT_SUCCESS(rc))
3018	{
3019	/*
3020	* We require a full resync now.
3021	*/
3022	for (unsigned i=0;i<pVM->cCPUs;i++)
3023	{
3024	PVMCPU pVCpu = &pVM->aCpus[i];
3025	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
3026	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3027
3028	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL;
3029	}
3030
3031	pgmR3HandlerPhysicalUpdateAll(pVM);
3032
3033	for (unsigned i=0;i<pVM->cCPUs;i++)
3034	{
3035	PVMCPU pVCpu = &pVM->aCpus[i];
3036
3037	/*
3038	* Change the paging mode.
3039	*/
3040	rc = PGMR3ChangeMode(pVM, pVCpu, pVCpu->pgm.s.enmGuestMode);
3041
3042	/* Restore pVM->pgm.s.GCPhysCR3. */
3043	Assert(pVCpu->pgm.s.GCPhysCR3 == NIL_RTGCPHYS);
3044	RTGCPHYS GCPhysCR3 = CPUMGetGuestCR3(pVCpu);
3045	if ( pVCpu->pgm.s.enmGuestMode == PGMMODE_PAE
3046	\|\| pVCpu->pgm.s.enmGuestMode == PGMMODE_PAE_NX
3047	\|\| pVCpu->pgm.s.enmGuestMode == PGMMODE_AMD64
3048	\|\| pVCpu->pgm.s.enmGuestMode == PGMMODE_AMD64_NX)
3049	GCPhysCR3 = (GCPhysCR3 & X86_CR3_PAE_PAGE_MASK);
3050	else
3051	GCPhysCR3 = (GCPhysCR3 & X86_CR3_PAGE_MASK);
3052	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
3053	}
3054	}
3055
3056	return rc;
3057	}
3058
3059
3060	/**
3061	* Show paging mode.
3062	*
3063	* @param pVM VM Handle.
3064	* @param pHlp The info helpers.
3065	* @param pszArgs "all" (default), "guest", "shadow" or "host".
3066	*/
3067	static DECLCALLBACK(void) pgmR3InfoMode(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
3068	{
3069	/* digest argument. */
3070	bool fGuest, fShadow, fHost;
3071	if (pszArgs)
3072	pszArgs = RTStrStripL(pszArgs);
3073	if (!pszArgs \|\| !*pszArgs \|\| strstr(pszArgs, "all"))
3074	fShadow = fHost = fGuest = true;
3075	else
3076	{
3077	fShadow = fHost = fGuest = false;
3078	if (strstr(pszArgs, "guest"))
3079	fGuest = true;
3080	if (strstr(pszArgs, "shadow"))
3081	fShadow = true;
3082	if (strstr(pszArgs, "host"))
3083	fHost = true;
3084	}
3085
3086	/** @todo SMP support! */
3087	/* print info. */
3088	if (fGuest)
3089	pHlp->pfnPrintf(pHlp, "Guest paging mode: %s, changed %RU64 times, A20 %s\n",
3090	PGMGetModeName(pVM->aCpus[0].pgm.s.enmGuestMode), pVM->aCpus[0].pgm.s.cGuestModeChanges.c,
3091	pVM->aCpus[0].pgm.s.fA20Enabled ? "enabled" : "disabled");
3092	if (fShadow)
3093	pHlp->pfnPrintf(pHlp, "Shadow paging mode: %s\n", PGMGetModeName(pVM->aCpus[0].pgm.s.enmShadowMode));
3094	if (fHost)
3095	{
3096	const char *psz;
3097	switch (pVM->pgm.s.enmHostMode)
3098	{
3099	case SUPPAGINGMODE_INVALID: psz = "invalid"; break;
3100	case SUPPAGINGMODE_32_BIT: psz = "32-bit"; break;
3101	case SUPPAGINGMODE_32_BIT_GLOBAL: psz = "32-bit+G"; break;
3102	case SUPPAGINGMODE_PAE: psz = "PAE"; break;
3103	case SUPPAGINGMODE_PAE_GLOBAL: psz = "PAE+G"; break;
3104	case SUPPAGINGMODE_PAE_NX: psz = "PAE+NX"; break;
3105	case SUPPAGINGMODE_PAE_GLOBAL_NX: psz = "PAE+G+NX"; break;
3106	case SUPPAGINGMODE_AMD64: psz = "AMD64"; break;
3107	case SUPPAGINGMODE_AMD64_GLOBAL: psz = "AMD64+G"; break;
3108	case SUPPAGINGMODE_AMD64_NX: psz = "AMD64+NX"; break;
3109	case SUPPAGINGMODE_AMD64_GLOBAL_NX: psz = "AMD64+G+NX"; break;
3110	default: psz = "unknown"; break;
3111	}
3112	pHlp->pfnPrintf(pHlp, "Host paging mode: %s\n", psz);
3113	}
3114	}
3115
3116
3117	/**
3118	* Dump registered MMIO ranges to the log.
3119	*
3120	* @param pVM VM Handle.
3121	* @param pHlp The info helpers.
3122	* @param pszArgs Arguments, ignored.
3123	*/
3124	static DECLCALLBACK(void) pgmR3PhysInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
3125	{
3126	NOREF(pszArgs);
3127	pHlp->pfnPrintf(pHlp,
3128	"RAM ranges (pVM=%p)\n"
3129	"%.s %.s\n",
3130	pVM,
3131	sizeof(RTGCPHYS) * 4 + 1, "GC Phys Range ",
3132	sizeof(RTHCPTR) * 2, "pvHC ");
3133
3134	for (PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
3135	pHlp->pfnPrintf(pHlp,
3136	"%RGp-%RGp %RHv %s\n",
3137	pCur->GCPhys,
3138	pCur->GCPhysLast,
3139	pCur->pvR3,
3140	pCur->pszDesc);
3141	}
3142
3143	/**
3144	* Dump the page directory to the log.
3145	*
3146	* @param pVM VM Handle.
3147	* @param pHlp The info helpers.
3148	* @param pszArgs Arguments, ignored.
3149	*/
3150	static DECLCALLBACK(void) pgmR3InfoCr3(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
3151	{
3152	/** @todo SMP support!! */
3153	PVMCPU pVCpu = &pVM->aCpus[0];
3154
3155	/** @todo fix this! Convert the PGMR3DumpHierarchyHC functions to do guest stuff. */
3156	/* Big pages supported? */
3157	const bool fPSE = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
3158
3159	/* Global pages supported? */
3160	const bool fPGE = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PGE);
3161
3162	NOREF(pszArgs);
3163
3164	/*
3165	* Get page directory addresses.
3166	*/
3167	PX86PD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3168	Assert(pPDSrc);
3169	Assert(PGMPhysGCPhys2R3PtrAssert(pVM, (RTGCPHYS)(CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK), sizeof(*pPDSrc)) == pPDSrc);
3170
3171	/*
3172	* Iterate the page directory.
3173	*/
3174	for (unsigned iPD = 0; iPD < RT_ELEMENTS(pPDSrc->a); iPD++)
3175	{
3176	X86PDE PdeSrc = pPDSrc->a[iPD];
3177	if (PdeSrc.n.u1Present)
3178	{
3179	if (PdeSrc.b.u1Size && fPSE)
3180	pHlp->pfnPrintf(pHlp,
3181	"%04X - %RGp P=%d U=%d RW=%d G=%d - BIG\n",
3182	iPD,
3183	pgmGstGet4MBPhysPage(&pVM->pgm.s, PdeSrc),
3184	PdeSrc.b.u1Present, PdeSrc.b.u1User, PdeSrc.b.u1Write, PdeSrc.b.u1Global && fPGE);
3185	else
3186	pHlp->pfnPrintf(pHlp,
3187	"%04X - %RGp P=%d U=%d RW=%d [G=%d]\n",
3188	iPD,
3189	(RTGCPHYS)(PdeSrc.u & X86_PDE_PG_MASK),
3190	PdeSrc.n.u1Present, PdeSrc.n.u1User, PdeSrc.n.u1Write, PdeSrc.b.u1Global && fPGE);
3191	}
3192	}
3193	}
3194
3195
3196	/**
3197	* Service a VMMCALLRING3_PGM_LOCK call.
3198	*
3199	* @returns VBox status code.
3200	* @param pVM The VM handle.
3201	*/
3202	VMMR3DECL(int) PGMR3LockCall(PVM pVM)
3203	{
3204	int rc = PDMR3CritSectEnterEx(&pVM->pgm.s.CritSect, true /* fHostCall */);
3205	AssertRC(rc);
3206	return rc;
3207	}
3208
3209
3210	/**
3211	* Converts a PGMMODE value to a PGM_TYPE_* \#define.
3212	*
3213	* @returns PGM_TYPE_*.
3214	* @param pgmMode The mode value to convert.
3215	*/
3216	DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
3217	{
3218	switch (pgmMode)
3219	{
3220	case PGMMODE_REAL: return PGM_TYPE_REAL;
3221	case PGMMODE_PROTECTED: return PGM_TYPE_PROT;
3222	case PGMMODE_32_BIT: return PGM_TYPE_32BIT;
3223	case PGMMODE_PAE:
3224	case PGMMODE_PAE_NX: return PGM_TYPE_PAE;
3225	case PGMMODE_AMD64:
3226	case PGMMODE_AMD64_NX: return PGM_TYPE_AMD64;
3227	case PGMMODE_NESTED: return PGM_TYPE_NESTED;
3228	case PGMMODE_EPT: return PGM_TYPE_EPT;
3229	default:
3230	AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
3231	}
3232	}
3233
3234
3235	/**
3236	* Gets the index into the paging mode data array of a SHW+GST mode.
3237	*
3238	* @returns PGM::paPagingData index.
3239	* @param uShwType The shadow paging mode type.
3240	* @param uGstType The guest paging mode type.
3241	*/
3242	DECLINLINE(unsigned) pgmModeDataIndex(unsigned uShwType, unsigned uGstType)
3243	{
3244	Assert(uShwType >= PGM_TYPE_32BIT && uShwType <= PGM_TYPE_MAX);
3245	Assert(uGstType >= PGM_TYPE_REAL && uGstType <= PGM_TYPE_AMD64);
3246	return (uShwType - PGM_TYPE_32BIT) * (PGM_TYPE_AMD64 - PGM_TYPE_REAL + 1)
3247	+ (uGstType - PGM_TYPE_REAL);
3248	}
3249
3250
3251	/**
3252	* Gets the index into the paging mode data array of a SHW+GST mode.
3253	*
3254	* @returns PGM::paPagingData index.
3255	* @param enmShw The shadow paging mode.
3256	* @param enmGst The guest paging mode.
3257	*/
3258	DECLINLINE(unsigned) pgmModeDataIndexByMode(PGMMODE enmShw, PGMMODE enmGst)
3259	{
3260	Assert(enmShw >= PGMMODE_32_BIT && enmShw <= PGMMODE_MAX);
3261	Assert(enmGst > PGMMODE_INVALID && enmGst < PGMMODE_MAX);
3262	return pgmModeDataIndex(pgmModeToType(enmShw), pgmModeToType(enmGst));
3263	}
3264
3265
3266	/**
3267	* Calculates the max data index.
3268	* @returns The number of entries in the paging data array.
3269	*/
3270	DECLINLINE(unsigned) pgmModeDataMaxIndex(void)
3271	{
3272	return pgmModeDataIndex(PGM_TYPE_MAX, PGM_TYPE_AMD64) + 1;
3273	}
3274
3275
3276	/**
3277	* Initializes the paging mode data kept in PGM::paModeData.
3278	*
3279	* @param pVM The VM handle.
3280	* @param fResolveGCAndR0 Indicate whether or not GC and Ring-0 symbols can be resolved now.
3281	* This is used early in the init process to avoid trouble with PDM
3282	* not being initialized yet.
3283	*/
3284	static int pgmR3ModeDataInit(PVM pVM, bool fResolveGCAndR0)
3285	{
3286	PPGMMODEDATA pModeData;
3287	int rc;
3288
3289	/*
3290	* Allocate the array on the first call.
3291	*/
3292	if (!pVM->pgm.s.paModeData)
3293	{
3294	pVM->pgm.s.paModeData = (PPGMMODEDATA)MMR3HeapAllocZ(pVM, MM_TAG_PGM, sizeof(PGMMODEDATA) * pgmModeDataMaxIndex());
3295	AssertReturn(pVM->pgm.s.paModeData, VERR_NO_MEMORY);
3296	}
3297
3298	/*
3299	* Initialize the array entries.
3300	*/
3301	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_32BIT, PGM_TYPE_REAL)];
3302	pModeData->uShwType = PGM_TYPE_32BIT;
3303	pModeData->uGstType = PGM_TYPE_REAL;
3304	rc = PGM_SHW_NAME_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3305	rc = PGM_GST_NAME_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3306	rc = PGM_BTH_NAME_32BIT_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3307
3308	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_32BIT, PGMMODE_PROTECTED)];
3309	pModeData->uShwType = PGM_TYPE_32BIT;
3310	pModeData->uGstType = PGM_TYPE_PROT;
3311	rc = PGM_SHW_NAME_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3312	rc = PGM_GST_NAME_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3313	rc = PGM_BTH_NAME_32BIT_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3314
3315	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_32BIT, PGM_TYPE_32BIT)];
3316	pModeData->uShwType = PGM_TYPE_32BIT;
3317	pModeData->uGstType = PGM_TYPE_32BIT;
3318	rc = PGM_SHW_NAME_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3319	rc = PGM_GST_NAME_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3320	rc = PGM_BTH_NAME_32BIT_32BIT(InitData)(pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3321
3322	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_PAE, PGM_TYPE_REAL)];
3323	pModeData->uShwType = PGM_TYPE_PAE;
3324	pModeData->uGstType = PGM_TYPE_REAL;
3325	rc = PGM_SHW_NAME_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3326	rc = PGM_GST_NAME_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3327	rc = PGM_BTH_NAME_PAE_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3328
3329	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_PAE, PGM_TYPE_PROT)];
3330	pModeData->uShwType = PGM_TYPE_PAE;
3331	pModeData->uGstType = PGM_TYPE_PROT;
3332	rc = PGM_SHW_NAME_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3333	rc = PGM_GST_NAME_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3334	rc = PGM_BTH_NAME_PAE_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3335
3336	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_PAE, PGM_TYPE_32BIT)];
3337	pModeData->uShwType = PGM_TYPE_PAE;
3338	pModeData->uGstType = PGM_TYPE_32BIT;
3339	rc = PGM_SHW_NAME_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3340	rc = PGM_GST_NAME_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3341	rc = PGM_BTH_NAME_PAE_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3342
3343	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_PAE, PGM_TYPE_PAE)];
3344	pModeData->uShwType = PGM_TYPE_PAE;
3345	pModeData->uGstType = PGM_TYPE_PAE;
3346	rc = PGM_SHW_NAME_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3347	rc = PGM_GST_NAME_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3348	rc = PGM_BTH_NAME_PAE_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3349
3350	#ifdef VBOX_WITH_64_BITS_GUESTS
3351	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_AMD64, PGM_TYPE_AMD64)];
3352	pModeData->uShwType = PGM_TYPE_AMD64;
3353	pModeData->uGstType = PGM_TYPE_AMD64;
3354	rc = PGM_SHW_NAME_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3355	rc = PGM_GST_NAME_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3356	rc = PGM_BTH_NAME_AMD64_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3357	#endif
3358
3359	/* The nested paging mode. */
3360	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_REAL)];
3361	pModeData->uShwType = PGM_TYPE_NESTED;
3362	pModeData->uGstType = PGM_TYPE_REAL;
3363	rc = PGM_GST_NAME_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3364	rc = PGM_BTH_NAME_NESTED_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3365
3366	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGMMODE_PROTECTED)];
3367	pModeData->uShwType = PGM_TYPE_NESTED;
3368	pModeData->uGstType = PGM_TYPE_PROT;
3369	rc = PGM_GST_NAME_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3370	rc = PGM_BTH_NAME_NESTED_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3371
3372	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_32BIT)];
3373	pModeData->uShwType = PGM_TYPE_NESTED;
3374	pModeData->uGstType = PGM_TYPE_32BIT;
3375	rc = PGM_GST_NAME_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3376	rc = PGM_BTH_NAME_NESTED_32BIT(InitData)(pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3377
3378	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_PAE)];
3379	pModeData->uShwType = PGM_TYPE_NESTED;
3380	pModeData->uGstType = PGM_TYPE_PAE;
3381	rc = PGM_GST_NAME_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3382	rc = PGM_BTH_NAME_NESTED_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3383
3384	#ifdef VBOX_WITH_64_BITS_GUESTS
3385	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_AMD64)];
3386	pModeData->uShwType = PGM_TYPE_NESTED;
3387	pModeData->uGstType = PGM_TYPE_AMD64;
3388	rc = PGM_GST_NAME_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3389	rc = PGM_BTH_NAME_NESTED_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3390	#endif
3391
3392	/* The shadow part of the nested callback mode depends on the host paging mode (AMD-V only). */
3393	switch (pVM->pgm.s.enmHostMode)
3394	{
3395	#if HC_ARCH_BITS == 32
3396	case SUPPAGINGMODE_32_BIT:
3397	case SUPPAGINGMODE_32_BIT_GLOBAL:
3398	for (unsigned i = PGM_TYPE_REAL; i <= PGM_TYPE_PAE; i++)
3399	{
3400	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, i)];
3401	rc = PGM_SHW_NAME_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3402	}
3403	# ifdef VBOX_WITH_64_BITS_GUESTS
3404	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_AMD64)];
3405	rc = PGM_SHW_NAME_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3406	# endif
3407	break;
3408
3409	case SUPPAGINGMODE_PAE:
3410	case SUPPAGINGMODE_PAE_NX:
3411	case SUPPAGINGMODE_PAE_GLOBAL:
3412	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3413	for (unsigned i = PGM_TYPE_REAL; i <= PGM_TYPE_PAE; i++)
3414	{
3415	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, i)];
3416	rc = PGM_SHW_NAME_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3417	}
3418	# ifdef VBOX_WITH_64_BITS_GUESTS
3419	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_AMD64)];
3420	rc = PGM_SHW_NAME_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3421	# endif
3422	break;
3423	#endif /* HC_ARCH_BITS == 32 */
3424
3425	#if HC_ARCH_BITS == 64 \|\| defined(RT_OS_DARWIN)
3426	case SUPPAGINGMODE_AMD64:
3427	case SUPPAGINGMODE_AMD64_GLOBAL:
3428	case SUPPAGINGMODE_AMD64_NX:
3429	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3430	# ifdef VBOX_WITH_64_BITS_GUESTS
3431	for (unsigned i = PGM_TYPE_REAL; i <= PGM_TYPE_AMD64; i++)
3432	# else
3433	for (unsigned i = PGM_TYPE_REAL; i <= PGM_TYPE_PAE; i++)
3434	# endif
3435	{
3436	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, i)];
3437	rc = PGM_SHW_NAME_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3438	}
3439	break;
3440	#endif /* HC_ARCH_BITS == 64 \|\| RT_OS_DARWIN */
3441
3442	default:
3443	AssertFailed();
3444	break;
3445	}
3446
3447	/* Extended paging (EPT) / Intel VT-x */
3448	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_REAL)];
3449	pModeData->uShwType = PGM_TYPE_EPT;
3450	pModeData->uGstType = PGM_TYPE_REAL;
3451	rc = PGM_SHW_NAME_EPT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3452	rc = PGM_GST_NAME_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3453	rc = PGM_BTH_NAME_EPT_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3454
3455	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_PROT)];
3456	pModeData->uShwType = PGM_TYPE_EPT;
3457	pModeData->uGstType = PGM_TYPE_PROT;
3458	rc = PGM_SHW_NAME_EPT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3459	rc = PGM_GST_NAME_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3460	rc = PGM_BTH_NAME_EPT_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3461
3462	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_32BIT)];
3463	pModeData->uShwType = PGM_TYPE_EPT;
3464	pModeData->uGstType = PGM_TYPE_32BIT;
3465	rc = PGM_SHW_NAME_EPT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3466	rc = PGM_GST_NAME_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3467	rc = PGM_BTH_NAME_EPT_32BIT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3468
3469	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_PAE)];
3470	pModeData->uShwType = PGM_TYPE_EPT;
3471	pModeData->uGstType = PGM_TYPE_PAE;
3472	rc = PGM_SHW_NAME_EPT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3473	rc = PGM_GST_NAME_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3474	rc = PGM_BTH_NAME_EPT_PAE(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3475
3476	#ifdef VBOX_WITH_64_BITS_GUESTS
3477	pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_AMD64)];
3478	pModeData->uShwType = PGM_TYPE_EPT;
3479	pModeData->uGstType = PGM_TYPE_AMD64;
3480	rc = PGM_SHW_NAME_EPT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3481	rc = PGM_GST_NAME_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3482	rc = PGM_BTH_NAME_EPT_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
3483	#endif
3484	return VINF_SUCCESS;
3485	}
3486
3487
3488	/**
3489	* Switch to different (or relocated in the relocate case) mode data.
3490	*
3491	* @param pVM The VM handle.
3492	* @param pVCpu The VMCPU to operate on.
3493	* @param enmShw The the shadow paging mode.
3494	* @param enmGst The the guest paging mode.
3495	*/
3496	static void pgmR3ModeDataSwitch(PVM pVM, PVMCPU pVCpu, PGMMODE enmShw, PGMMODE enmGst)
3497	{
3498	PPGMMODEDATA pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndexByMode(enmShw, enmGst)];
3499
3500	Assert(pModeData->uGstType == pgmModeToType(enmGst));
3501	Assert(pModeData->uShwType == pgmModeToType(enmShw));
3502
3503	/* shadow */
3504	pVCpu->pgm.s.pfnR3ShwRelocate = pModeData->pfnR3ShwRelocate;
3505	pVCpu->pgm.s.pfnR3ShwExit = pModeData->pfnR3ShwExit;
3506	pVCpu->pgm.s.pfnR3ShwGetPage = pModeData->pfnR3ShwGetPage;
3507	Assert(pVCpu->pgm.s.pfnR3ShwGetPage);
3508	pVCpu->pgm.s.pfnR3ShwModifyPage = pModeData->pfnR3ShwModifyPage;
3509
3510	pVCpu->pgm.s.pfnRCShwGetPage = pModeData->pfnRCShwGetPage;
3511	pVCpu->pgm.s.pfnRCShwModifyPage = pModeData->pfnRCShwModifyPage;
3512
3513	pVCpu->pgm.s.pfnR0ShwGetPage = pModeData->pfnR0ShwGetPage;
3514	pVCpu->pgm.s.pfnR0ShwModifyPage = pModeData->pfnR0ShwModifyPage;
3515
3516
3517	/* guest */
3518	pVCpu->pgm.s.pfnR3GstRelocate = pModeData->pfnR3GstRelocate;
3519	pVCpu->pgm.s.pfnR3GstExit = pModeData->pfnR3GstExit;
3520	pVCpu->pgm.s.pfnR3GstGetPage = pModeData->pfnR3GstGetPage;
3521	Assert(pVCpu->pgm.s.pfnR3GstGetPage);
3522	pVCpu->pgm.s.pfnR3GstModifyPage = pModeData->pfnR3GstModifyPage;
3523	pVCpu->pgm.s.pfnR3GstGetPDE = pModeData->pfnR3GstGetPDE;
3524	pVCpu->pgm.s.pfnRCGstGetPage = pModeData->pfnRCGstGetPage;
3525	pVCpu->pgm.s.pfnRCGstModifyPage = pModeData->pfnRCGstModifyPage;
3526	pVCpu->pgm.s.pfnRCGstGetPDE = pModeData->pfnRCGstGetPDE;
3527	pVCpu->pgm.s.pfnR0GstGetPage = pModeData->pfnR0GstGetPage;
3528	pVCpu->pgm.s.pfnR0GstModifyPage = pModeData->pfnR0GstModifyPage;
3529	pVCpu->pgm.s.pfnR0GstGetPDE = pModeData->pfnR0GstGetPDE;
3530
3531	/* both */
3532	pVCpu->pgm.s.pfnR3BthRelocate = pModeData->pfnR3BthRelocate;
3533	pVCpu->pgm.s.pfnR3BthInvalidatePage = pModeData->pfnR3BthInvalidatePage;
3534	pVCpu->pgm.s.pfnR3BthSyncCR3 = pModeData->pfnR3BthSyncCR3;
3535	Assert(pVCpu->pgm.s.pfnR3BthSyncCR3);
3536	pVCpu->pgm.s.pfnR3BthSyncPage = pModeData->pfnR3BthSyncPage;
3537	pVCpu->pgm.s.pfnR3BthPrefetchPage = pModeData->pfnR3BthPrefetchPage;
3538	pVCpu->pgm.s.pfnR3BthVerifyAccessSyncPage = pModeData->pfnR3BthVerifyAccessSyncPage;
3539	#ifdef VBOX_STRICT
3540	pVCpu->pgm.s.pfnR3BthAssertCR3 = pModeData->pfnR3BthAssertCR3;
3541	#endif
3542	pVCpu->pgm.s.pfnR3BthMapCR3 = pModeData->pfnR3BthMapCR3;
3543	pVCpu->pgm.s.pfnR3BthUnmapCR3 = pModeData->pfnR3BthUnmapCR3;
3544
3545	pVCpu->pgm.s.pfnRCBthTrap0eHandler = pModeData->pfnRCBthTrap0eHandler;
3546	pVCpu->pgm.s.pfnRCBthInvalidatePage = pModeData->pfnRCBthInvalidatePage;
3547	pVCpu->pgm.s.pfnRCBthSyncCR3 = pModeData->pfnRCBthSyncCR3;
3548	pVCpu->pgm.s.pfnRCBthSyncPage = pModeData->pfnRCBthSyncPage;
3549	pVCpu->pgm.s.pfnRCBthPrefetchPage = pModeData->pfnRCBthPrefetchPage;
3550	pVCpu->pgm.s.pfnRCBthVerifyAccessSyncPage = pModeData->pfnRCBthVerifyAccessSyncPage;
3551	#ifdef VBOX_STRICT
3552	pVCpu->pgm.s.pfnRCBthAssertCR3 = pModeData->pfnRCBthAssertCR3;
3553	#endif
3554	pVCpu->pgm.s.pfnRCBthMapCR3 = pModeData->pfnRCBthMapCR3;
3555	pVCpu->pgm.s.pfnRCBthUnmapCR3 = pModeData->pfnRCBthUnmapCR3;
3556
3557	pVCpu->pgm.s.pfnR0BthTrap0eHandler = pModeData->pfnR0BthTrap0eHandler;
3558	pVCpu->pgm.s.pfnR0BthInvalidatePage = pModeData->pfnR0BthInvalidatePage;
3559	pVCpu->pgm.s.pfnR0BthSyncCR3 = pModeData->pfnR0BthSyncCR3;
3560	pVCpu->pgm.s.pfnR0BthSyncPage = pModeData->pfnR0BthSyncPage;
3561	pVCpu->pgm.s.pfnR0BthPrefetchPage = pModeData->pfnR0BthPrefetchPage;
3562	pVCpu->pgm.s.pfnR0BthVerifyAccessSyncPage = pModeData->pfnR0BthVerifyAccessSyncPage;
3563	#ifdef VBOX_STRICT
3564	pVCpu->pgm.s.pfnR0BthAssertCR3 = pModeData->pfnR0BthAssertCR3;
3565	#endif
3566	pVCpu->pgm.s.pfnR0BthMapCR3 = pModeData->pfnR0BthMapCR3;
3567	pVCpu->pgm.s.pfnR0BthUnmapCR3 = pModeData->pfnR0BthUnmapCR3;
3568	}
3569
3570
3571	/**
3572	* Calculates the shadow paging mode.
3573	*
3574	* @returns The shadow paging mode.
3575	* @param pVM VM handle.
3576	* @param enmGuestMode The guest mode.
3577	* @param enmHostMode The host mode.
3578	* @param enmShadowMode The current shadow mode.
3579	* @param penmSwitcher Where to store the switcher to use.
3580	* VMMSWITCHER_INVALID means no change.
3581	*/
3582	static PGMMODE pgmR3CalcShadowMode(PVM pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode, VMMSWITCHER *penmSwitcher)
3583	{
3584	VMMSWITCHER enmSwitcher = VMMSWITCHER_INVALID;
3585	switch (enmGuestMode)
3586	{
3587	/*
3588	* When switching to real or protected mode we don't change
3589	* anything since it's likely that we'll switch back pretty soon.
3590	*
3591	* During pgmR3InitPaging we'll end up here with PGMMODE_INVALID
3592	* and is supposed to determine which shadow paging and switcher to
3593	* use during init.
3594	*/
3595	case PGMMODE_REAL:
3596	case PGMMODE_PROTECTED:
3597	if ( enmShadowMode != PGMMODE_INVALID
3598	&& !HWACCMIsEnabled(pVM) /* always switch in hwaccm mode! */)
3599	break; /* (no change) */
3600
3601	switch (enmHostMode)
3602	{
3603	case SUPPAGINGMODE_32_BIT:
3604	case SUPPAGINGMODE_32_BIT_GLOBAL:
3605	enmShadowMode = PGMMODE_32_BIT;
3606	enmSwitcher = VMMSWITCHER_32_TO_32;
3607	break;
3608
3609	case SUPPAGINGMODE_PAE:
3610	case SUPPAGINGMODE_PAE_NX:
3611	case SUPPAGINGMODE_PAE_GLOBAL:
3612	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3613	enmShadowMode = PGMMODE_PAE;
3614	enmSwitcher = VMMSWITCHER_PAE_TO_PAE;
3615	#ifdef DEBUG_bird
3616	if (RTEnvExist("VBOX_32BIT"))
3617	{
3618	enmShadowMode = PGMMODE_32_BIT;
3619	enmSwitcher = VMMSWITCHER_PAE_TO_32;
3620	}
3621	#endif
3622	break;
3623
3624	case SUPPAGINGMODE_AMD64:
3625	case SUPPAGINGMODE_AMD64_GLOBAL:
3626	case SUPPAGINGMODE_AMD64_NX:
3627	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3628	enmShadowMode = PGMMODE_PAE;
3629	enmSwitcher = VMMSWITCHER_AMD64_TO_PAE;
3630	#ifdef DEBUG_bird
3631	if (RTEnvExist("VBOX_32BIT"))
3632	{
3633	enmShadowMode = PGMMODE_32_BIT;
3634	enmSwitcher = VMMSWITCHER_AMD64_TO_32;
3635	}
3636	#endif
3637	break;
3638
3639	default: AssertMsgFailed(("enmHostMode=%d\n", enmHostMode)); break;
3640	}
3641	break;
3642
3643	case PGMMODE_32_BIT:
3644	switch (enmHostMode)
3645	{
3646	case SUPPAGINGMODE_32_BIT:
3647	case SUPPAGINGMODE_32_BIT_GLOBAL:
3648	enmShadowMode = PGMMODE_32_BIT;
3649	enmSwitcher = VMMSWITCHER_32_TO_32;
3650	break;
3651
3652	case SUPPAGINGMODE_PAE:
3653	case SUPPAGINGMODE_PAE_NX:
3654	case SUPPAGINGMODE_PAE_GLOBAL:
3655	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3656	enmShadowMode = PGMMODE_PAE;
3657	enmSwitcher = VMMSWITCHER_PAE_TO_PAE;
3658	#ifdef DEBUG_bird
3659	if (RTEnvExist("VBOX_32BIT"))
3660	{
3661	enmShadowMode = PGMMODE_32_BIT;
3662	enmSwitcher = VMMSWITCHER_PAE_TO_32;
3663	}
3664	#endif
3665	break;
3666
3667	case SUPPAGINGMODE_AMD64:
3668	case SUPPAGINGMODE_AMD64_GLOBAL:
3669	case SUPPAGINGMODE_AMD64_NX:
3670	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3671	enmShadowMode = PGMMODE_PAE;
3672	enmSwitcher = VMMSWITCHER_AMD64_TO_PAE;
3673	#ifdef DEBUG_bird
3674	if (RTEnvExist("VBOX_32BIT"))
3675	{
3676	enmShadowMode = PGMMODE_32_BIT;
3677	enmSwitcher = VMMSWITCHER_AMD64_TO_32;
3678	}
3679	#endif
3680	break;
3681
3682	default: AssertMsgFailed(("enmHostMode=%d\n", enmHostMode)); break;
3683	}
3684	break;
3685
3686	case PGMMODE_PAE:
3687	case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
3688	switch (enmHostMode)
3689	{
3690	case SUPPAGINGMODE_32_BIT:
3691	case SUPPAGINGMODE_32_BIT_GLOBAL:
3692	enmShadowMode = PGMMODE_PAE;
3693	enmSwitcher = VMMSWITCHER_32_TO_PAE;
3694	break;
3695
3696	case SUPPAGINGMODE_PAE:
3697	case SUPPAGINGMODE_PAE_NX:
3698	case SUPPAGINGMODE_PAE_GLOBAL:
3699	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3700	enmShadowMode = PGMMODE_PAE;
3701	enmSwitcher = VMMSWITCHER_PAE_TO_PAE;
3702	break;
3703
3704	case SUPPAGINGMODE_AMD64:
3705	case SUPPAGINGMODE_AMD64_GLOBAL:
3706	case SUPPAGINGMODE_AMD64_NX:
3707	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3708	enmShadowMode = PGMMODE_PAE;
3709	enmSwitcher = VMMSWITCHER_AMD64_TO_PAE;
3710	break;
3711
3712	default: AssertMsgFailed(("enmHostMode=%d\n", enmHostMode)); break;
3713	}
3714	break;
3715
3716	case PGMMODE_AMD64:
3717	case PGMMODE_AMD64_NX:
3718	switch (enmHostMode)
3719	{
3720	case SUPPAGINGMODE_32_BIT:
3721	case SUPPAGINGMODE_32_BIT_GLOBAL:
3722	enmShadowMode = PGMMODE_AMD64;
3723	enmSwitcher = VMMSWITCHER_32_TO_AMD64;
3724	break;
3725
3726	case SUPPAGINGMODE_PAE:
3727	case SUPPAGINGMODE_PAE_NX:
3728	case SUPPAGINGMODE_PAE_GLOBAL:
3729	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3730	enmShadowMode = PGMMODE_AMD64;
3731	enmSwitcher = VMMSWITCHER_PAE_TO_AMD64;
3732	break;
3733
3734	case SUPPAGINGMODE_AMD64:
3735	case SUPPAGINGMODE_AMD64_GLOBAL:
3736	case SUPPAGINGMODE_AMD64_NX:
3737	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3738	enmShadowMode = PGMMODE_AMD64;
3739	enmSwitcher = VMMSWITCHER_AMD64_TO_AMD64;
3740	break;
3741
3742	default: AssertMsgFailed(("enmHostMode=%d\n", enmHostMode)); break;
3743	}
3744	break;
3745
3746
3747	default:
3748	AssertReleaseMsgFailed(("enmGuestMode=%d\n", enmGuestMode));
3749	return PGMMODE_INVALID;
3750	}
3751	/* Override the shadow mode is nested paging is active. */
3752	if (HWACCMIsNestedPagingActive(pVM))
3753	enmShadowMode = HWACCMGetShwPagingMode(pVM);
3754
3755	*penmSwitcher = enmSwitcher;
3756	return enmShadowMode;
3757	}
3758
3759
3760	/**
3761	* Performs the actual mode change.
3762	* This is called by PGMChangeMode and pgmR3InitPaging().
3763	*
3764	* @returns VBox status code. May suspend or power off the VM on error, but this
3765	* will trigger using FFs and not status codes.
3766	*
3767	* @param pVM VM handle.
3768	* @param pVCpu The VMCPU to operate on.
3769	* @param enmGuestMode The new guest mode. This is assumed to be different from
3770	* the current mode.
3771	*/
3772	VMMR3DECL(int) PGMR3ChangeMode(PVM pVM, PVMCPU pVCpu, PGMMODE enmGuestMode)
3773	{
3774	Log(("PGMR3ChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
3775	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
3776
3777	/*
3778	* Calc the shadow mode and switcher.
3779	*/
3780	VMMSWITCHER enmSwitcher;
3781	PGMMODE enmShadowMode = pgmR3CalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode, &enmSwitcher);
3782	if (enmSwitcher != VMMSWITCHER_INVALID)
3783	{
3784	/*
3785	* Select new switcher.
3786	*/
3787	int rc = VMMR3SelectSwitcher(pVM, enmSwitcher);
3788	if (RT_FAILURE(rc))
3789	{
3790	AssertReleaseMsgFailed(("VMMR3SelectSwitcher(%d) -> %Rrc\n", enmSwitcher, rc));
3791	return rc;
3792	}
3793	}
3794
3795	/*
3796	* Exit old mode(s).
3797	*/
3798	/* shadow */
3799	if (enmShadowMode != pVCpu->pgm.s.enmShadowMode)
3800	{
3801	LogFlow(("PGMR3ChangeMode: Shadow mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
3802	if (PGM_SHW_PFN(Exit, pVCpu))
3803	{
3804	int rc = PGM_SHW_PFN(Exit, pVCpu)(pVCpu);
3805	if (RT_FAILURE(rc))
3806	{
3807	AssertMsgFailed(("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc));
3808	return rc;
3809	}
3810	}
3811
3812	}
3813	else
3814	LogFlow(("PGMR3ChangeMode: Shadow mode remains: %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
3815
3816	/* guest */
3817	if (PGM_GST_PFN(Exit, pVCpu))
3818	{
3819	int rc = PGM_GST_PFN(Exit, pVCpu)(pVCpu);
3820	if (RT_FAILURE(rc))
3821	{
3822	AssertMsgFailed(("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc));
3823	return rc;
3824	}
3825	}
3826
3827	/*
3828	* Load new paging mode data.
3829	*/
3830	pgmR3ModeDataSwitch(pVM, pVCpu, enmShadowMode, enmGuestMode);
3831
3832	/*
3833	* Enter new shadow mode (if changed).
3834	*/
3835	if (enmShadowMode != pVCpu->pgm.s.enmShadowMode)
3836	{
3837	int rc;
3838	pVCpu->pgm.s.enmShadowMode = enmShadowMode;
3839	switch (enmShadowMode)
3840	{
3841	case PGMMODE_32_BIT:
3842	rc = PGM_SHW_NAME_32BIT(Enter)(pVCpu);
3843	break;
3844	case PGMMODE_PAE:
3845	case PGMMODE_PAE_NX:
3846	rc = PGM_SHW_NAME_PAE(Enter)(pVCpu);
3847	break;
3848	case PGMMODE_AMD64:
3849	case PGMMODE_AMD64_NX:
3850	rc = PGM_SHW_NAME_AMD64(Enter)(pVCpu);
3851	break;
3852	case PGMMODE_NESTED:
3853	rc = PGM_SHW_NAME_NESTED(Enter)(pVCpu);
3854	break;
3855	case PGMMODE_EPT:
3856	rc = PGM_SHW_NAME_EPT(Enter)(pVCpu);
3857	break;
3858	case PGMMODE_REAL:
3859	case PGMMODE_PROTECTED:
3860	default:
3861	AssertReleaseMsgFailed(("enmShadowMode=%d\n", enmShadowMode));
3862	return VERR_INTERNAL_ERROR;
3863	}
3864	if (RT_FAILURE(rc))
3865	{
3866	AssertReleaseMsgFailed(("Entering enmShadowMode=%d failed: %Rrc\n", enmShadowMode, rc));
3867	pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID;
3868	return rc;
3869	}
3870	}
3871
3872	/*
3873	* Always flag the necessary updates
3874	*/
3875	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3876
3877	/*
3878	* Enter the new guest and shadow+guest modes.
3879	*/
3880	int rc = -1;
3881	int rc2 = -1;
3882	RTGCPHYS GCPhysCR3 = NIL_RTGCPHYS;
3883	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3884	switch (enmGuestMode)
3885	{
3886	case PGMMODE_REAL:
3887	rc = PGM_GST_NAME_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
3888	switch (pVCpu->pgm.s.enmShadowMode)
3889	{
3890	case PGMMODE_32_BIT:
3891	rc2 = PGM_BTH_NAME_32BIT_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
3892	break;
3893	case PGMMODE_PAE:
3894	case PGMMODE_PAE_NX:
3895	rc2 = PGM_BTH_NAME_PAE_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
3896	break;
3897	case PGMMODE_NESTED:
3898	rc2 = PGM_BTH_NAME_NESTED_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
3899	break;
3900	case PGMMODE_EPT:
3901	rc2 = PGM_BTH_NAME_EPT_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
3902	break;
3903	case PGMMODE_AMD64:
3904	case PGMMODE_AMD64_NX:
3905	AssertMsgFailed(("Should use PAE shadow mode!\n"));
3906	default: AssertFailed(); break;
3907	}
3908	break;
3909
3910	case PGMMODE_PROTECTED:
3911	rc = PGM_GST_NAME_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
3912	switch (pVCpu->pgm.s.enmShadowMode)
3913	{
3914	case PGMMODE_32_BIT:
3915	rc2 = PGM_BTH_NAME_32BIT_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
3916	break;
3917	case PGMMODE_PAE:
3918	case PGMMODE_PAE_NX:
3919	rc2 = PGM_BTH_NAME_PAE_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
3920	break;
3921	case PGMMODE_NESTED:
3922	rc2 = PGM_BTH_NAME_NESTED_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
3923	break;
3924	case PGMMODE_EPT:
3925	rc2 = PGM_BTH_NAME_EPT_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
3926	break;
3927	case PGMMODE_AMD64:
3928	case PGMMODE_AMD64_NX:
3929	AssertMsgFailed(("Should use PAE shadow mode!\n"));
3930	default: AssertFailed(); break;
3931	}
3932	break;
3933
3934	case PGMMODE_32_BIT:
3935	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
3936	rc = PGM_GST_NAME_32BIT(Enter)(pVCpu, GCPhysCR3);
3937	switch (pVCpu->pgm.s.enmShadowMode)
3938	{
3939	case PGMMODE_32_BIT:
3940	rc2 = PGM_BTH_NAME_32BIT_32BIT(Enter)(pVCpu, GCPhysCR3);
3941	break;
3942	case PGMMODE_PAE:
3943	case PGMMODE_PAE_NX:
3944	rc2 = PGM_BTH_NAME_PAE_32BIT(Enter)(pVCpu, GCPhysCR3);
3945	break;
3946	case PGMMODE_NESTED:
3947	rc2 = PGM_BTH_NAME_NESTED_32BIT(Enter)(pVCpu, GCPhysCR3);
3948	break;
3949	case PGMMODE_EPT:
3950	rc2 = PGM_BTH_NAME_EPT_32BIT(Enter)(pVCpu, GCPhysCR3);
3951	break;
3952	case PGMMODE_AMD64:
3953	case PGMMODE_AMD64_NX:
3954	AssertMsgFailed(("Should use PAE shadow mode!\n"));
3955	default: AssertFailed(); break;
3956	}
3957	break;
3958
3959	case PGMMODE_PAE_NX:
3960	case PGMMODE_PAE:
3961	{
3962	uint32_t u32Dummy, u32Features;
3963
3964	CPUMGetGuestCpuId(pVCpu, 1, &u32Dummy, &u32Dummy, &u32Dummy, &u32Features);
3965	if (!(u32Features & X86_CPUID_FEATURE_EDX_PAE))
3966	return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
3967	N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (General/Advanced)"));
3968
3969	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
3970	rc = PGM_GST_NAME_PAE(Enter)(pVCpu, GCPhysCR3);
3971	switch (pVCpu->pgm.s.enmShadowMode)
3972	{
3973	case PGMMODE_PAE:
3974	case PGMMODE_PAE_NX:
3975	rc2 = PGM_BTH_NAME_PAE_PAE(Enter)(pVCpu, GCPhysCR3);
3976	break;
3977	case PGMMODE_NESTED:
3978	rc2 = PGM_BTH_NAME_NESTED_PAE(Enter)(pVCpu, GCPhysCR3);
3979	break;
3980	case PGMMODE_EPT:
3981	rc2 = PGM_BTH_NAME_EPT_PAE(Enter)(pVCpu, GCPhysCR3);
3982	break;
3983	case PGMMODE_32_BIT:
3984	case PGMMODE_AMD64:
3985	case PGMMODE_AMD64_NX:
3986	AssertMsgFailed(("Should use PAE shadow mode!\n"));
3987	default: AssertFailed(); break;
3988	}
3989	break;
3990	}
3991
3992	#ifdef VBOX_WITH_64_BITS_GUESTS
3993	case PGMMODE_AMD64_NX:
3994	case PGMMODE_AMD64:
3995	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & UINT64_C(0xfffffffffffff000); /** @todo define this mask! */
3996	rc = PGM_GST_NAME_AMD64(Enter)(pVCpu, GCPhysCR3);
3997	switch (pVCpu->pgm.s.enmShadowMode)
3998	{
3999	case PGMMODE_AMD64:
4000	case PGMMODE_AMD64_NX:
4001	rc2 = PGM_BTH_NAME_AMD64_AMD64(Enter)(pVCpu, GCPhysCR3);
4002	break;
4003	case PGMMODE_NESTED:
4004	rc2 = PGM_BTH_NAME_NESTED_AMD64(Enter)(pVCpu, GCPhysCR3);
4005	break;
4006	case PGMMODE_EPT:
4007	rc2 = PGM_BTH_NAME_EPT_AMD64(Enter)(pVCpu, GCPhysCR3);
4008	break;
4009	case PGMMODE_32_BIT:
4010	case PGMMODE_PAE:
4011	case PGMMODE_PAE_NX:
4012	AssertMsgFailed(("Should use AMD64 shadow mode!\n"));
4013	default: AssertFailed(); break;
4014	}
4015	break;
4016	#endif
4017
4018	default:
4019	AssertReleaseMsgFailed(("enmGuestMode=%d\n", enmGuestMode));
4020	rc = VERR_NOT_IMPLEMENTED;
4021	break;
4022	}
4023
4024	/* status codes. */
4025	AssertRC(rc);
4026	AssertRC(rc2);
4027	if (RT_SUCCESS(rc))
4028	{
4029	rc = rc2;
4030	if (RT_SUCCESS(rc)) /* no informational status codes. */
4031	rc = VINF_SUCCESS;
4032	}
4033
4034	/* Notify HWACCM as well. */
4035	HWACCMR3PagingModeChanged(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
4036	return rc;
4037	}
4038
4039	/**
4040	* Release the pgm lock if owned by the current VCPU
4041	*
4042	* @param pVM The VM to operate on.
4043	*/
4044	VMMR3DECL(void) PGMR3ReleaseOwnedLocks(PVM pVM)
4045	{
4046	while (PDMCritSectIsOwner(&pVM->pgm.s.CritSect))
4047	PDMCritSectLeave(&pVM->pgm.s.CritSect);
4048	}
4049
4050	/**
4051	* Called by pgmPoolFlushAllInt prior to flushing the pool.
4052	*
4053	* @returns VBox status code, fully asserted.
4054	* @param pVM The VM handle.
4055	* @param pVCpu The VMCPU to operate on.
4056	*/
4057	int pgmR3ExitShadowModeBeforePoolFlush(PVM pVM, PVMCPU pVCpu)
4058	{
4059	/* Unmap the old CR3 value before flushing everything. */
4060	int rc = PGM_BTH_PFN(UnmapCR3, pVCpu)(pVCpu);
4061	AssertRC(rc);
4062
4063	/* Exit the current shadow paging mode as well; nested paging and EPT use a root CR3 which will get flushed here. */
4064	rc = PGM_SHW_PFN(Exit, pVCpu)(pVCpu);
4065	AssertRC(rc);
4066	Assert(pVCpu->pgm.s.pShwPageCR3R3 == NULL);
4067	return rc;
4068	}
4069
4070
4071	/**
4072	* Called by pgmPoolFlushAllInt after flushing the pool.
4073	*
4074	* @returns VBox status code, fully asserted.
4075	* @param pVM The VM handle.
4076	* @param pVCpu The VMCPU to operate on.
4077	*/
4078	int pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu)
4079	{
4080	pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID;
4081	int rc = PGMR3ChangeMode(pVM, pVCpu, PGMGetGuestMode(pVCpu));
4082	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4083	AssertRCReturn(rc, rc);
4084	AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
4085
4086	Assert(pVCpu->pgm.s.pShwPageCR3R3 != NULL);
4087	AssertMsg( pVCpu->pgm.s.enmShadowMode >= PGMMODE_NESTED
4088	\|\| CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu),
4089	("%RHp != %RHp %s\n", (RTHCPHYS)CPUMGetHyperCR3(pVCpu), PGMGetHyperCR3(pVCpu), PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
4090	return rc;
4091	}
4092
4093
4094	/**
4095	* Dumps a PAE shadow page table.
4096	*
4097	* @returns VBox status code (VINF_SUCCESS).
4098	* @param pVM The VM handle.
4099	* @param pPT Pointer to the page table.
4100	* @param u64Address The virtual address of the page table starts.
4101	* @param fLongMode Set if this a long mode table; clear if it's a legacy mode table.
4102	* @param cMaxDepth The maxium depth.
4103	* @param pHlp Pointer to the output functions.
4104	*/
4105	static int pgmR3DumpHierarchyHCPaePT(PVM pVM, PX86PTPAE pPT, uint64_t u64Address, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp)
4106	{
4107	for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
4108	{
4109	X86PTEPAE Pte = pPT->a[i];
4110	if (Pte.n.u1Present)
4111	{
4112	pHlp->pfnPrintf(pHlp,
4113	fLongMode /P R S A D G WT CD AT NX 4M a p ? /
4114	? "%016llx 3 \| P %c %c %c %c %c %s %s %s %s 4K %c%c%c %016llx\n"
4115	: "%08llx 2 \| P %c %c %c %c %c %s %s %s %s 4K %c%c%c %016llx\n",
4116	u64Address + ((uint64_t)i << X86_PT_PAE_SHIFT),
4117	Pte.n.u1Write ? 'W' : 'R',
4118	Pte.n.u1User ? 'U' : 'S',
4119	Pte.n.u1Accessed ? 'A' : '-',
4120	Pte.n.u1Dirty ? 'D' : '-',
4121	Pte.n.u1Global ? 'G' : '-',
4122	Pte.n.u1WriteThru ? "WT" : "--",
4123	Pte.n.u1CacheDisable? "CD" : "--",
4124	Pte.n.u1PAT ? "AT" : "--",
4125	Pte.n.u1NoExecute ? "NX" : "--",
4126	Pte.u & PGM_PTFLAGS_TRACK_DIRTY ? 'd' : '-',
4127	Pte.u & RT_BIT(10) ? '1' : '0',
4128	Pte.u & PGM_PTFLAGS_CSAM_VALIDATED? 'v' : '-',
4129	Pte.u & X86_PTE_PAE_PG_MASK);
4130	}
4131	}
4132	return VINF_SUCCESS;
4133	}
4134
4135
4136	/**
4137	* Dumps a PAE shadow page directory table.
4138	*
4139	* @returns VBox status code (VINF_SUCCESS).
4140	* @param pVM The VM handle.
4141	* @param HCPhys The physical address of the page directory table.
4142	* @param u64Address The virtual address of the page table starts.
4143	* @param cr4 The CR4, PSE is currently used.
4144	* @param fLongMode Set if this a long mode table; clear if it's a legacy mode table.
4145	* @param cMaxDepth The maxium depth.
4146	* @param pHlp Pointer to the output functions.
4147	*/
4148	static int pgmR3DumpHierarchyHCPaePD(PVM pVM, RTHCPHYS HCPhys, uint64_t u64Address, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp)
4149	{
4150	PX86PDPAE pPD = (PX86PDPAE)MMPagePhys2Page(pVM, HCPhys);
4151	if (!pPD)
4152	{
4153	pHlp->pfnPrintf(pHlp, "%0*llx error! Page directory at HCPhys=%RHp was not found in the page pool!\n",
4154	fLongMode ? 16 : 8, u64Address, HCPhys);
4155	return VERR_INVALID_PARAMETER;
4156	}
4157	const bool fBigPagesSupported = fLongMode \|\| !!(cr4 & X86_CR4_PSE);
4158
4159	int rc = VINF_SUCCESS;
4160	for (unsigned i = 0; i < RT_ELEMENTS(pPD->a); i++)
4161	{
4162	X86PDEPAE Pde = pPD->a[i];
4163	if (Pde.n.u1Present)
4164	{
4165	if (fBigPagesSupported && Pde.b.u1Size)
4166	pHlp->pfnPrintf(pHlp,
4167	fLongMode /P R S A D G WT CD AT NX 4M a p ? /
4168	? "%016llx 2 \| P %c %c %c %c %c %s %s %s %s 4M %c%c%c %016llx\n"
4169	: "%08llx 1 \| P %c %c %c %c %c %s %s %s %s 4M %c%c%c %016llx\n",
4170	u64Address + ((uint64_t)i << X86_PD_PAE_SHIFT),
4171	Pde.b.u1Write ? 'W' : 'R',
4172	Pde.b.u1User ? 'U' : 'S',
4173	Pde.b.u1Accessed ? 'A' : '-',
4174	Pde.b.u1Dirty ? 'D' : '-',
4175	Pde.b.u1Global ? 'G' : '-',
4176	Pde.b.u1WriteThru ? "WT" : "--",
4177	Pde.b.u1CacheDisable? "CD" : "--",
4178	Pde.b.u1PAT ? "AT" : "--",
4179	Pde.b.u1NoExecute ? "NX" : "--",
4180	Pde.u & RT_BIT_64(9) ? '1' : '0',
4181	Pde.u & PGM_PDFLAGS_MAPPING ? 'm' : '-',
4182	Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
4183	Pde.u & X86_PDE_PAE_PG_MASK);
4184	else
4185	{
4186	pHlp->pfnPrintf(pHlp,
4187	fLongMode /P R S A D G WT CD AT NX 4M a p ? /
4188	? "%016llx 2 \| P %c %c %c %c %c %s %s .. %s 4K %c%c%c %016llx\n"
4189	: "%08llx 1 \| P %c %c %c %c %c %s %s .. %s 4K %c%c%c %016llx\n",
4190	u64Address + ((uint64_t)i << X86_PD_PAE_SHIFT),
4191	Pde.n.u1Write ? 'W' : 'R',
4192	Pde.n.u1User ? 'U' : 'S',
4193	Pde.n.u1Accessed ? 'A' : '-',
4194	Pde.n.u1Reserved0 ? '?' : '.', /* ignored */
4195	Pde.n.u1Reserved1 ? '?' : '.', /* ignored */
4196	Pde.n.u1WriteThru ? "WT" : "--",
4197	Pde.n.u1CacheDisable? "CD" : "--",
4198	Pde.n.u1NoExecute ? "NX" : "--",
4199	Pde.u & RT_BIT_64(9) ? '1' : '0',
4200	Pde.u & PGM_PDFLAGS_MAPPING ? 'm' : '-',
4201	Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
4202	Pde.u & X86_PDE_PAE_PG_MASK);
4203	if (cMaxDepth >= 1)
4204	{
4205	/** @todo what about using the page pool for mapping PTs? */
4206	uint64_t u64AddressPT = u64Address + ((uint64_t)i << X86_PD_PAE_SHIFT);
4207	RTHCPHYS HCPhysPT = Pde.u & X86_PDE_PAE_PG_MASK;
4208	PX86PTPAE pPT = NULL;
4209	if (!(Pde.u & PGM_PDFLAGS_MAPPING))
4210	pPT = (PX86PTPAE)MMPagePhys2Page(pVM, HCPhysPT);
4211	else
4212	{
4213	for (PPGMMAPPING pMap = pVM->pgm.s.pMappingsR3; pMap; pMap = pMap->pNextR3)
4214	{
4215	uint64_t off = u64AddressPT - pMap->GCPtr;
4216	if (off < pMap->cb)
4217	{
4218	const int iPDE = (uint32_t)(off >> X86_PD_SHIFT);
4219	const int iSub = (int)((off >> X86_PD_PAE_SHIFT) & 1); /* MSC is a pain sometimes */
4220	if ((iSub ? pMap->aPTs[iPDE].HCPhysPaePT1 : pMap->aPTs[iPDE].HCPhysPaePT0) != HCPhysPT)
4221	pHlp->pfnPrintf(pHlp, "%0*llx error! Mapping error! PT %d has HCPhysPT=%RHp not %RHp is in the PD.\n",
4222	fLongMode ? 16 : 8, u64AddressPT, iPDE,
4223	iSub ? pMap->aPTs[iPDE].HCPhysPaePT1 : pMap->aPTs[iPDE].HCPhysPaePT0, HCPhysPT);
4224	pPT = &pMap->aPTs[iPDE].paPaePTsR3[iSub];
4225	}
4226	}
4227	}
4228	int rc2 = VERR_INVALID_PARAMETER;
4229	if (pPT)
4230	rc2 = pgmR3DumpHierarchyHCPaePT(pVM, pPT, u64AddressPT, fLongMode, cMaxDepth - 1, pHlp);
4231	else
4232	pHlp->pfnPrintf(pHlp, "%0*llx error! Page table at HCPhys=%RHp was not found in the page pool!\n",
4233	fLongMode ? 16 : 8, u64AddressPT, HCPhysPT);
4234	if (rc2 < rc && RT_SUCCESS(rc))
4235	rc = rc2;
4236	}
4237	}
4238	}
4239	}
4240	return rc;
4241	}
4242
4243
4244	/**
4245	* Dumps a PAE shadow page directory pointer table.
4246	*
4247	* @returns VBox status code (VINF_SUCCESS).
4248	* @param pVM The VM handle.
4249	* @param HCPhys The physical address of the page directory pointer table.
4250	* @param u64Address The virtual address of the page table starts.
4251	* @param cr4 The CR4, PSE is currently used.
4252	* @param fLongMode Set if this a long mode table; clear if it's a legacy mode table.
4253	* @param cMaxDepth The maxium depth.
4254	* @param pHlp Pointer to the output functions.
4255	*/
4256	static int pgmR3DumpHierarchyHCPaePDPT(PVM pVM, RTHCPHYS HCPhys, uint64_t u64Address, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp)
4257	{
4258	PX86PDPT pPDPT = (PX86PDPT)MMPagePhys2Page(pVM, HCPhys);
4259	if (!pPDPT)
4260	{
4261	pHlp->pfnPrintf(pHlp, "%0*llx error! Page directory pointer table at HCPhys=%RHp was not found in the page pool!\n",
4262	fLongMode ? 16 : 8, u64Address, HCPhys);
4263	return VERR_INVALID_PARAMETER;
4264	}
4265
4266	int rc = VINF_SUCCESS;
4267	const unsigned c = fLongMode ? RT_ELEMENTS(pPDPT->a) : X86_PG_PAE_PDPE_ENTRIES;
4268	for (unsigned i = 0; i < c; i++)
4269	{
4270	X86PDPE Pdpe = pPDPT->a[i];
4271	if (Pdpe.n.u1Present)
4272	{
4273	if (fLongMode)
4274	pHlp->pfnPrintf(pHlp, /P R S A D G WT CD AT NX 4M a p ? /
4275	"%016llx 1 \| P %c %c %c %c %c %s %s %s %s .. %c%c%c %016llx\n",
4276	u64Address + ((uint64_t)i << X86_PDPT_SHIFT),
4277	Pdpe.lm.u1Write ? 'W' : 'R',
4278	Pdpe.lm.u1User ? 'U' : 'S',
4279	Pdpe.lm.u1Accessed ? 'A' : '-',
4280	Pdpe.lm.u3Reserved & 1? '?' : '.', /* ignored */
4281	Pdpe.lm.u3Reserved & 4? '!' : '.', /* mbz */
4282	Pdpe.lm.u1WriteThru ? "WT" : "--",
4283	Pdpe.lm.u1CacheDisable? "CD" : "--",
4284	Pdpe.lm.u3Reserved & 2? "!" : "..",/* mbz */
4285	Pdpe.lm.u1NoExecute ? "NX" : "--",
4286	Pdpe.u & RT_BIT(9) ? '1' : '0',
4287	Pdpe.u & PGM_PLXFLAGS_PERMANENT ? 'p' : '-',
4288	Pdpe.u & RT_BIT(11) ? '1' : '0',
4289	Pdpe.u & X86_PDPE_PG_MASK);
4290	else
4291	pHlp->pfnPrintf(pHlp, /P G WT CD AT NX 4M a p ? /
4292	"%08x 0 \| P %c %s %s %s %s .. %c%c%c %016llx\n",
4293	i << X86_PDPT_SHIFT,
4294	Pdpe.n.u4Reserved & 1? '!' : '.', /* mbz */
4295	Pdpe.n.u4Reserved & 4? '!' : '.', /* mbz */
4296	Pdpe.n.u1WriteThru ? "WT" : "--",
4297	Pdpe.n.u1CacheDisable? "CD" : "--",
4298	Pdpe.n.u4Reserved & 2? "!" : "..",/* mbz */
4299	Pdpe.u & RT_BIT(9) ? '1' : '0',
4300	Pdpe.u & PGM_PLXFLAGS_PERMANENT ? 'p' : '-',
4301	Pdpe.u & RT_BIT(11) ? '1' : '0',
4302	Pdpe.u & X86_PDPE_PG_MASK);
4303	if (cMaxDepth >= 1)
4304	{
4305	int rc2 = pgmR3DumpHierarchyHCPaePD(pVM, Pdpe.u & X86_PDPE_PG_MASK, u64Address + ((uint64_t)i << X86_PDPT_SHIFT),
4306	cr4, fLongMode, cMaxDepth - 1, pHlp);
4307	if (rc2 < rc && RT_SUCCESS(rc))
4308	rc = rc2;
4309	}
4310	}
4311	}
4312	return rc;
4313	}
4314
4315
4316	/**
4317	* Dumps a 32-bit shadow page table.
4318	*
4319	* @returns VBox status code (VINF_SUCCESS).
4320	* @param pVM The VM handle.
4321	* @param HCPhys The physical address of the table.
4322	* @param cr4 The CR4, PSE is currently used.
4323	* @param cMaxDepth The maxium depth.
4324	* @param pHlp Pointer to the output functions.
4325	*/
4326	static int pgmR3DumpHierarchyHcPaePML4(PVM pVM, RTHCPHYS HCPhys, uint32_t cr4, unsigned cMaxDepth, PCDBGFINFOHLP pHlp)
4327	{
4328	PX86PML4 pPML4 = (PX86PML4)MMPagePhys2Page(pVM, HCPhys);
4329	if (!pPML4)
4330	{
4331	pHlp->pfnPrintf(pHlp, "Page map level 4 at HCPhys=%RHp was not found in the page pool!\n", HCPhys);
4332	return VERR_INVALID_PARAMETER;
4333	}
4334
4335	int rc = VINF_SUCCESS;
4336	for (unsigned i = 0; i < RT_ELEMENTS(pPML4->a); i++)
4337	{
4338	X86PML4E Pml4e = pPML4->a[i];
4339	if (Pml4e.n.u1Present)
4340	{
4341	uint64_t u64Address = ((uint64_t)i << X86_PML4_SHIFT) \| (((uint64_t)i >> (X86_PML4_SHIFT - X86_PDPT_SHIFT - 1)) * 0xffff000000000000ULL);
4342	pHlp->pfnPrintf(pHlp, /P R S A D G WT CD AT NX 4M a p ? /
4343	"%016llx 0 \| P %c %c %c %c %c %s %s %s %s .. %c%c%c %016llx\n",
4344	u64Address,
4345	Pml4e.n.u1Write ? 'W' : 'R',
4346	Pml4e.n.u1User ? 'U' : 'S',
4347	Pml4e.n.u1Accessed ? 'A' : '-',
4348	Pml4e.n.u3Reserved & 1? '?' : '.', /* ignored */
4349	Pml4e.n.u3Reserved & 4? '!' : '.', /* mbz */
4350	Pml4e.n.u1WriteThru ? "WT" : "--",
4351	Pml4e.n.u1CacheDisable? "CD" : "--",
4352	Pml4e.n.u3Reserved & 2? "!" : "..",/* mbz */
4353	Pml4e.n.u1NoExecute ? "NX" : "--",
4354	Pml4e.u & RT_BIT(9) ? '1' : '0',
4355	Pml4e.u & PGM_PLXFLAGS_PERMANENT ? 'p' : '-',
4356	Pml4e.u & RT_BIT(11) ? '1' : '0',
4357	Pml4e.u & X86_PML4E_PG_MASK);
4358
4359	if (cMaxDepth >= 1)
4360	{
4361	int rc2 = pgmR3DumpHierarchyHCPaePDPT(pVM, Pml4e.u & X86_PML4E_PG_MASK, u64Address, cr4, true, cMaxDepth - 1, pHlp);
4362	if (rc2 < rc && RT_SUCCESS(rc))
4363	rc = rc2;
4364	}
4365	}
4366	}
4367	return rc;
4368	}
4369
4370
4371	/**
4372	* Dumps a 32-bit shadow page table.
4373	*
4374	* @returns VBox status code (VINF_SUCCESS).
4375	* @param pVM The VM handle.
4376	* @param pPT Pointer to the page table.
4377	* @param u32Address The virtual address this table starts at.
4378	* @param pHlp Pointer to the output functions.
4379	*/
4380	int pgmR3DumpHierarchyHC32BitPT(PVM pVM, PX86PT pPT, uint32_t u32Address, PCDBGFINFOHLP pHlp)
4381	{
4382	for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
4383	{
4384	X86PTE Pte = pPT->a[i];
4385	if (Pte.n.u1Present)
4386	{
4387	pHlp->pfnPrintf(pHlp, /P R S A D G WT CD AT NX 4M a m d /
4388	"%08x 1 \| P %c %c %c %c %c %s %s %s .. 4K %c%c%c %08x\n",
4389	u32Address + (i << X86_PT_SHIFT),
4390	Pte.n.u1Write ? 'W' : 'R',
4391	Pte.n.u1User ? 'U' : 'S',
4392	Pte.n.u1Accessed ? 'A' : '-',
4393	Pte.n.u1Dirty ? 'D' : '-',
4394	Pte.n.u1Global ? 'G' : '-',
4395	Pte.n.u1WriteThru ? "WT" : "--",
4396	Pte.n.u1CacheDisable? "CD" : "--",
4397	Pte.n.u1PAT ? "AT" : "--",
4398	Pte.u & PGM_PTFLAGS_TRACK_DIRTY ? 'd' : '-',
4399	Pte.u & RT_BIT(10) ? '1' : '0',
4400	Pte.u & PGM_PTFLAGS_CSAM_VALIDATED ? 'v' : '-',
4401	Pte.u & X86_PDE_PG_MASK);
4402	}
4403	}
4404	return VINF_SUCCESS;
4405	}
4406
4407
4408	/**
4409	* Dumps a 32-bit shadow page directory and page tables.
4410	*
4411	* @returns VBox status code (VINF_SUCCESS).
4412	* @param pVM The VM handle.
4413	* @param cr3 The root of the hierarchy.
4414	* @param cr4 The CR4, PSE is currently used.
4415	* @param cMaxDepth How deep into the hierarchy the dumper should go.
4416	* @param pHlp Pointer to the output functions.
4417	*/
4418	int pgmR3DumpHierarchyHC32BitPD(PVM pVM, uint32_t cr3, uint32_t cr4, unsigned cMaxDepth, PCDBGFINFOHLP pHlp)
4419	{
4420	PX86PD pPD = (PX86PD)MMPagePhys2Page(pVM, cr3 & X86_CR3_PAGE_MASK);
4421	if (!pPD)
4422	{
4423	pHlp->pfnPrintf(pHlp, "Page directory at %#x was not found in the page pool!\n", cr3 & X86_CR3_PAGE_MASK);
4424	return VERR_INVALID_PARAMETER;
4425	}
4426
4427	int rc = VINF_SUCCESS;
4428	for (unsigned i = 0; i < RT_ELEMENTS(pPD->a); i++)
4429	{
4430	X86PDE Pde = pPD->a[i];
4431	if (Pde.n.u1Present)
4432	{
4433	const uint32_t u32Address = i << X86_PD_SHIFT;
4434	if ((cr4 & X86_CR4_PSE) && Pde.b.u1Size)
4435	pHlp->pfnPrintf(pHlp, /P R S A D G WT CD AT NX 4M a m d /
4436	"%08x 0 \| P %c %c %c %c %c %s %s %s .. 4M %c%c%c %08x\n",
4437	u32Address,
4438	Pde.b.u1Write ? 'W' : 'R',
4439	Pde.b.u1User ? 'U' : 'S',
4440	Pde.b.u1Accessed ? 'A' : '-',
4441	Pde.b.u1Dirty ? 'D' : '-',
4442	Pde.b.u1Global ? 'G' : '-',
4443	Pde.b.u1WriteThru ? "WT" : "--",
4444	Pde.b.u1CacheDisable? "CD" : "--",
4445	Pde.b.u1PAT ? "AT" : "--",
4446	Pde.u & RT_BIT_64(9) ? '1' : '0',
4447	Pde.u & PGM_PDFLAGS_MAPPING ? 'm' : '-',
4448	Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
4449	Pde.u & X86_PDE4M_PG_MASK);
4450	else
4451	{
4452	pHlp->pfnPrintf(pHlp, /P R S A D G WT CD AT NX 4M a m d /
4453	"%08x 0 \| P %c %c %c %c %c %s %s .. .. 4K %c%c%c %08x\n",
4454	u32Address,
4455	Pde.n.u1Write ? 'W' : 'R',
4456	Pde.n.u1User ? 'U' : 'S',
4457	Pde.n.u1Accessed ? 'A' : '-',
4458	Pde.n.u1Reserved0 ? '?' : '.', /* ignored */
4459	Pde.n.u1Reserved1 ? '?' : '.', /* ignored */
4460	Pde.n.u1WriteThru ? "WT" : "--",
4461	Pde.n.u1CacheDisable? "CD" : "--",
4462	Pde.u & RT_BIT_64(9) ? '1' : '0',
4463	Pde.u & PGM_PDFLAGS_MAPPING ? 'm' : '-',
4464	Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
4465	Pde.u & X86_PDE_PG_MASK);
4466	if (cMaxDepth >= 1)
4467	{
4468	/** @todo what about using the page pool for mapping PTs? */
4469	RTHCPHYS HCPhys = Pde.u & X86_PDE_PG_MASK;
4470	PX86PT pPT = NULL;
4471	if (!(Pde.u & PGM_PDFLAGS_MAPPING))
4472	pPT = (PX86PT)MMPagePhys2Page(pVM, HCPhys);
4473	else
4474	{
4475	for (PPGMMAPPING pMap = pVM->pgm.s.pMappingsR3; pMap; pMap = pMap->pNextR3)
4476	if (u32Address - pMap->GCPtr < pMap->cb)
4477	{
4478	int iPDE = (u32Address - pMap->GCPtr) >> X86_PD_SHIFT;
4479	if (pMap->aPTs[iPDE].HCPhysPT != HCPhys)
4480	pHlp->pfnPrintf(pHlp, "%08x error! Mapping error! PT %d has HCPhysPT=%RHp not %RHp is in the PD.\n",
4481	u32Address, iPDE, pMap->aPTs[iPDE].HCPhysPT, HCPhys);
4482	pPT = pMap->aPTs[iPDE].pPTR3;
4483	}
4484	}
4485	int rc2 = VERR_INVALID_PARAMETER;
4486	if (pPT)
4487	rc2 = pgmR3DumpHierarchyHC32BitPT(pVM, pPT, u32Address, pHlp);
4488	else
4489	pHlp->pfnPrintf(pHlp, "%08x error! Page table at %#x was not found in the page pool!\n", u32Address, HCPhys);
4490	if (rc2 < rc && RT_SUCCESS(rc))
4491	rc = rc2;
4492	}
4493	}
4494	}
4495	}
4496
4497	return rc;
4498	}
4499
4500
4501	/**
4502	* Dumps a 32-bit shadow page table.
4503	*
4504	* @returns VBox status code (VINF_SUCCESS).
4505	* @param pVM The VM handle.
4506	* @param pPT Pointer to the page table.
4507	* @param u32Address The virtual address this table starts at.
4508	* @param PhysSearch Address to search for.
4509	*/
4510	int pgmR3DumpHierarchyGC32BitPT(PVM pVM, PX86PT pPT, uint32_t u32Address, RTGCPHYS PhysSearch)
4511	{
4512	for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
4513	{
4514	X86PTE Pte = pPT->a[i];
4515	if (Pte.n.u1Present)
4516	{
4517	Log(( /P R S A D G WT CD AT NX 4M a m d /
4518	"%08x 1 \| P %c %c %c %c %c %s %s %s .. 4K %c%c%c %08x\n",
4519	u32Address + (i << X86_PT_SHIFT),
4520	Pte.n.u1Write ? 'W' : 'R',
4521	Pte.n.u1User ? 'U' : 'S',
4522	Pte.n.u1Accessed ? 'A' : '-',
4523	Pte.n.u1Dirty ? 'D' : '-',
4524	Pte.n.u1Global ? 'G' : '-',
4525	Pte.n.u1WriteThru ? "WT" : "--",
4526	Pte.n.u1CacheDisable? "CD" : "--",
4527	Pte.n.u1PAT ? "AT" : "--",
4528	Pte.u & PGM_PTFLAGS_TRACK_DIRTY ? 'd' : '-',
4529	Pte.u & RT_BIT(10) ? '1' : '0',
4530	Pte.u & PGM_PTFLAGS_CSAM_VALIDATED ? 'v' : '-',
4531	Pte.u & X86_PDE_PG_MASK));
4532
4533	if ((Pte.u & X86_PDE_PG_MASK) == PhysSearch)
4534	{
4535	uint64_t fPageShw = 0;
4536	RTHCPHYS pPhysHC = 0;
4537
4538	/** @todo SMP support!! */
4539	PGMShwGetPage(&pVM->aCpus[0], (RTGCPTR)(u32Address + (i << X86_PT_SHIFT)), &fPageShw, &pPhysHC);
4540	Log(("Found %RGp at %RGv -> flags=%llx\n", PhysSearch, (RTGCPTR)(u32Address + (i << X86_PT_SHIFT)), fPageShw));
4541	}
4542	}
4543	}
4544	return VINF_SUCCESS;
4545	}
4546
4547
4548	/**
4549	* Dumps a 32-bit guest page directory and page tables.
4550	*
4551	* @returns VBox status code (VINF_SUCCESS).
4552	* @param pVM The VM handle.
4553	* @param cr3 The root of the hierarchy.
4554	* @param cr4 The CR4, PSE is currently used.
4555	* @param PhysSearch Address to search for.
4556	*/
4557	VMMR3DECL(int) PGMR3DumpHierarchyGC(PVM pVM, uint64_t cr3, uint64_t cr4, RTGCPHYS PhysSearch)
4558	{
4559	bool fLongMode = false;
4560	const unsigned cch = fLongMode ? 16 : 8; NOREF(cch);
4561	PX86PD pPD = 0;
4562
4563	int rc = PGM_GCPHYS_2_PTR(pVM, cr3 & X86_CR3_PAGE_MASK, &pPD);
4564	if (RT_FAILURE(rc) \|\| !pPD)
4565	{
4566	Log(("Page directory at %#x was not found in the page pool!\n", cr3 & X86_CR3_PAGE_MASK));
4567	return VERR_INVALID_PARAMETER;
4568	}
4569
4570	Log(("cr3=%08x cr4=%08x%s\n"
4571	"%-*s P - Present\n"
4572	"%-*s \| R/W - Read (0) / Write (1)\n"
4573	"%-*s \| \| U/S - User (1) / Supervisor (0)\n"
4574	"%-*s \| \| \| A - Accessed\n"
4575	"%-*s \| \| \| \| D - Dirty\n"
4576	"%-*s \| \| \| \| \| G - Global\n"
4577	"%-*s \| \| \| \| \| \| WT - Write thru\n"
4578	"%-*s \| \| \| \| \| \| \| CD - Cache disable\n"
4579	"%-*s \| \| \| \| \| \| \| \| AT - Attribute table (PAT)\n"
4580	"%-*s \| \| \| \| \| \| \| \| \| NX - No execute (K8)\n"
4581	"%-*s \| \| \| \| \| \| \| \| \| \| 4K/4M/2M - Page size.\n"
4582	"%-*s \| \| \| \| \| \| \| \| \| \| \| AVL - a=allocated; m=mapping; d=track dirty;\n"
4583	"%-*s \| \| \| \| \| \| \| \| \| \| \| \| p=permanent; v=validated;\n"
4584	"%-*s Level \| \| \| \| \| \| \| \| \| \| \| \| Page\n"
4585	/* xxxx n **** P R S A D G WT CD AT NX 4M AVL xxxxxxxxxxxxx
4586	- W U - - - -- -- -- -- -- 010 */
4587	, cr3, cr4, fLongMode ? " Long Mode" : "",
4588	cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "",
4589	cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "Address"));
4590
4591	for (unsigned i = 0; i < RT_ELEMENTS(pPD->a); i++)
4592	{
4593	X86PDE Pde = pPD->a[i];
4594	if (Pde.n.u1Present)
4595	{
4596	const uint32_t u32Address = i << X86_PD_SHIFT;
4597
4598	if ((cr4 & X86_CR4_PSE) && Pde.b.u1Size)
4599	Log(( /P R S A D G WT CD AT NX 4M a m d /
4600	"%08x 0 \| P %c %c %c %c %c %s %s %s .. 4M %c%c%c %08x\n",
4601	u32Address,
4602	Pde.b.u1Write ? 'W' : 'R',
4603	Pde.b.u1User ? 'U' : 'S',
4604	Pde.b.u1Accessed ? 'A' : '-',
4605	Pde.b.u1Dirty ? 'D' : '-',
4606	Pde.b.u1Global ? 'G' : '-',
4607	Pde.b.u1WriteThru ? "WT" : "--",
4608	Pde.b.u1CacheDisable? "CD" : "--",
4609	Pde.b.u1PAT ? "AT" : "--",
4610	Pde.u & RT_BIT(9) ? '1' : '0',
4611	Pde.u & RT_BIT(10) ? '1' : '0',
4612	Pde.u & RT_BIT(11) ? '1' : '0',
4613	pgmGstGet4MBPhysPage(&pVM->pgm.s, Pde)));
4614	/** @todo PhysSearch */
4615	else
4616	{
4617	Log(( /P R S A D G WT CD AT NX 4M a m d /
4618	"%08x 0 \| P %c %c %c %c %c %s %s .. .. 4K %c%c%c %08x\n",
4619	u32Address,
4620	Pde.n.u1Write ? 'W' : 'R',
4621	Pde.n.u1User ? 'U' : 'S',
4622	Pde.n.u1Accessed ? 'A' : '-',
4623	Pde.n.u1Reserved0 ? '?' : '.', /* ignored */
4624	Pde.n.u1Reserved1 ? '?' : '.', /* ignored */
4625	Pde.n.u1WriteThru ? "WT" : "--",
4626	Pde.n.u1CacheDisable? "CD" : "--",
4627	Pde.u & RT_BIT(9) ? '1' : '0',
4628	Pde.u & RT_BIT(10) ? '1' : '0',
4629	Pde.u & RT_BIT(11) ? '1' : '0',
4630	Pde.u & X86_PDE_PG_MASK));
4631	////if (cMaxDepth >= 1)
4632	{
4633	/** @todo what about using the page pool for mapping PTs? */
4634	RTGCPHYS GCPhys = Pde.u & X86_PDE_PG_MASK;
4635	PX86PT pPT = NULL;
4636
4637	rc = PGM_GCPHYS_2_PTR(pVM, GCPhys, &pPT);
4638
4639	int rc2 = VERR_INVALID_PARAMETER;
4640	if (pPT)
4641	rc2 = pgmR3DumpHierarchyGC32BitPT(pVM, pPT, u32Address, PhysSearch);
4642	else
4643	Log(("%08x error! Page table at %#x was not found in the page pool!\n", u32Address, GCPhys));
4644	if (rc2 < rc && RT_SUCCESS(rc))
4645	rc = rc2;
4646	}
4647	}
4648	}
4649	}
4650
4651	return rc;
4652	}
4653
4654
4655	/**
4656	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
4657	*
4658	* @returns VBox status code (VINF_SUCCESS).
4659	* @param pVM The VM handle.
4660	* @param cr3 The root of the hierarchy.
4661	* @param cr4 The cr4, only PAE and PSE is currently used.
4662	* @param fLongMode Set if long mode, false if not long mode.
4663	* @param cMaxDepth Number of levels to dump.
4664	* @param pHlp Pointer to the output functions.
4665	*/
4666	VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint64_t cr3, uint64_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp)
4667	{
4668	if (!pHlp)
4669	pHlp = DBGFR3InfoLogHlp();
4670	if (!cMaxDepth)
4671	return VINF_SUCCESS;
4672	const unsigned cch = fLongMode ? 16 : 8;
4673	pHlp->pfnPrintf(pHlp,
4674	"cr3=%08x cr4=%08x%s\n"
4675	"%-*s P - Present\n"
4676	"%-*s \| R/W - Read (0) / Write (1)\n"
4677	"%-*s \| \| U/S - User (1) / Supervisor (0)\n"
4678	"%-*s \| \| \| A - Accessed\n"
4679	"%-*s \| \| \| \| D - Dirty\n"
4680	"%-*s \| \| \| \| \| G - Global\n"
4681	"%-*s \| \| \| \| \| \| WT - Write thru\n"
4682	"%-*s \| \| \| \| \| \| \| CD - Cache disable\n"
4683	"%-*s \| \| \| \| \| \| \| \| AT - Attribute table (PAT)\n"
4684	"%-*s \| \| \| \| \| \| \| \| \| NX - No execute (K8)\n"
4685	"%-*s \| \| \| \| \| \| \| \| \| \| 4K/4M/2M - Page size.\n"
4686	"%-*s \| \| \| \| \| \| \| \| \| \| \| AVL - a=allocated; m=mapping; d=track dirty;\n"
4687	"%-*s \| \| \| \| \| \| \| \| \| \| \| \| p=permanent; v=validated;\n"
4688	"%-*s Level \| \| \| \| \| \| \| \| \| \| \| \| Page\n"
4689	/* xxxx n **** P R S A D G WT CD AT NX 4M AVL xxxxxxxxxxxxx
4690	- W U - - - -- -- -- -- -- 010 */
4691	, cr3, cr4, fLongMode ? " Long Mode" : "",
4692	cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "",
4693	cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "Address");
4694	if (cr4 & X86_CR4_PAE)
4695	{
4696	if (fLongMode)
4697	return pgmR3DumpHierarchyHcPaePML4(pVM, cr3 & X86_CR3_PAGE_MASK, cr4, cMaxDepth, pHlp);
4698	return pgmR3DumpHierarchyHCPaePDPT(pVM, cr3 & X86_CR3_PAE_PAGE_MASK, 0, cr4, false, cMaxDepth, pHlp);
4699	}
4700	return pgmR3DumpHierarchyHC32BitPD(pVM, cr3 & X86_CR3_PAGE_MASK, cr4, cMaxDepth, pHlp);
4701	}
4702
4703	#ifdef VBOX_WITH_DEBUGGER
4704
4705	/**
4706	* The '.pgmram' command.
4707	*
4708	* @returns VBox status.
4709	* @param pCmd Pointer to the command descriptor (as registered).
4710	* @param pCmdHlp Pointer to command helper functions.
4711	* @param pVM Pointer to the current VM (if any).
4712	* @param paArgs Pointer to (readonly) array of arguments.
4713	* @param cArgs Number of arguments in the array.
4714	*/
4715	static DECLCALLBACK(int) pgmR3CmdRam(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
4716	{
4717	/*
4718	* Validate input.
4719	*/
4720	if (!pVM)
4721	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
4722	if (!pVM->pgm.s.pRamRangesRC)
4723	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Sorry, no Ram is registered.\n");
4724
4725	/*
4726	* Dump the ranges.
4727	*/
4728	int rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL, "From - To (incl) pvHC\n");
4729	PPGMRAMRANGE pRam;
4730	for (pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
4731	{
4732	rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL,
4733	"%RGp - %RGp %p\n",
4734	pRam->GCPhys, pRam->GCPhysLast, pRam->pvR3);
4735	if (RT_FAILURE(rc))
4736	return rc;
4737	}
4738
4739	return VINF_SUCCESS;
4740	}
4741
4742
4743	/**
4744	* The '.pgmmap' command.
4745	*
4746	* @returns VBox status.
4747	* @param pCmd Pointer to the command descriptor (as registered).
4748	* @param pCmdHlp Pointer to command helper functions.
4749	* @param pVM Pointer to the current VM (if any).
4750	* @param paArgs Pointer to (readonly) array of arguments.
4751	* @param cArgs Number of arguments in the array.
4752	*/
4753	static DECLCALLBACK(int) pgmR3CmdMap(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
4754	{
4755	/*
4756	* Validate input.
4757	*/
4758	if (!pVM)
4759	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
4760	if (!pVM->pgm.s.pMappingsR3)
4761	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Sorry, no mappings are registered.\n");
4762
4763	/*
4764	* Print message about the fixedness of the mappings.
4765	*/
4766	int rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL, pVM->pgm.s.fMappingsFixed ? "The mappings are FIXED.\n" : "The mappings are FLOATING.\n");
4767	if (RT_FAILURE(rc))
4768	return rc;
4769
4770	/*
4771	* Dump the ranges.
4772	*/
4773	PPGMMAPPING pCur;
4774	for (pCur = pVM->pgm.s.pMappingsR3; pCur; pCur = pCur->pNextR3)
4775	{
4776	rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL,
4777	"%08x - %08x %s\n",
4778	pCur->GCPtr, pCur->GCPtrLast, pCur->pszDesc);
4779	if (RT_FAILURE(rc))
4780	return rc;
4781	}
4782
4783	return VINF_SUCCESS;
4784	}
4785
4786
4787	/**
4788	* The '.pgmerror' and '.pgmerroroff' commands.
4789	*
4790	* @returns VBox status.
4791	* @param pCmd Pointer to the command descriptor (as registered).
4792	* @param pCmdHlp Pointer to command helper functions.
4793	* @param pVM Pointer to the current VM (if any).
4794	* @param paArgs Pointer to (readonly) array of arguments.
4795	* @param cArgs Number of arguments in the array.
4796	*/
4797	static DECLCALLBACK(int) pgmR3CmdError(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
4798	{
4799	/*
4800	* Validate input.
4801	*/
4802	if (!pVM)
4803	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
4804	AssertReturn(cArgs == 0 \|\| (cArgs == 1 && paArgs[0].enmType == DBGCVAR_TYPE_STRING),
4805	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: Hit bug in the parser.\n"));
4806
4807	if (!cArgs)
4808	{
4809	/*
4810	* Print the list of error injection locations with status.
4811	*/
4812	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "PGM error inject locations:\n");
4813	pCmdHlp->pfnPrintf(pCmdHlp, NULL, " handy - %RTbool\n", pVM->pgm.s.fErrInjHandyPages);
4814	}
4815	else
4816	{
4817
4818	/*
4819	* String switch on where to inject the error.
4820	*/
4821	bool const fNewState = !strcmp(pCmd->pszCmd, "pgmerror");
4822	const char *pszWhere = paArgs[0].u.pszString;
4823	if (!strcmp(pszWhere, "handy"))
4824	ASMAtomicWriteBool(&pVM->pgm.s.fErrInjHandyPages, fNewState);
4825	else
4826	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: Invalid 'where' value: %s.\n", pszWhere);
4827	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "done\n");
4828	}
4829	return VINF_SUCCESS;
4830	}
4831
4832
4833	/**
4834	* The '.pgmsync' command.
4835	*
4836	* @returns VBox status.
4837	* @param pCmd Pointer to the command descriptor (as registered).
4838	* @param pCmdHlp Pointer to command helper functions.
4839	* @param pVM Pointer to the current VM (if any).
4840	* @param paArgs Pointer to (readonly) array of arguments.
4841	* @param cArgs Number of arguments in the array.
4842	*/
4843	static DECLCALLBACK(int) pgmR3CmdSync(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
4844	{
4845	/** @todo SMP support */
4846	PVMCPU pVCpu = &pVM->aCpus[0];
4847
4848	/*
4849	* Validate input.
4850	*/
4851	if (!pVM)
4852	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
4853
4854	/*
4855	* Force page directory sync.
4856	*/
4857	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
4858
4859	int rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Forcing page directory sync.\n");
4860	if (RT_FAILURE(rc))
4861	return rc;
4862
4863	return VINF_SUCCESS;
4864	}
4865
4866
4867	#ifdef VBOX_STRICT
4868	/**
4869	* The '.pgmassertcr3' command.
4870	*
4871	* @returns VBox status.
4872	* @param pCmd Pointer to the command descriptor (as registered).
4873	* @param pCmdHlp Pointer to command helper functions.
4874	* @param pVM Pointer to the current VM (if any).
4875	* @param paArgs Pointer to (readonly) array of arguments.
4876	* @param cArgs Number of arguments in the array.
4877	*/
4878	static DECLCALLBACK(int) pgmR3CmdAssertCR3(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
4879	{
4880	/** @todo SMP support!! */
4881	PVMCPU pVCpu = &pVM->aCpus[0];
4882
4883	/*
4884	* Validate input.
4885	*/
4886	if (!pVM)
4887	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
4888
4889	int rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Checking shadow CR3 page tables for consistency.\n");
4890	if (RT_FAILURE(rc))
4891	return rc;
4892
4893	PGMAssertCR3(pVM, pVCpu, CPUMGetGuestCR3(pVCpu), CPUMGetGuestCR4(pVCpu));
4894
4895	return VINF_SUCCESS;
4896	}
4897	#endif /* VBOX_STRICT */
4898
4899
4900	/**
4901	* The '.pgmsyncalways' command.
4902	*
4903	* @returns VBox status.
4904	* @param pCmd Pointer to the command descriptor (as registered).
4905	* @param pCmdHlp Pointer to command helper functions.
4906	* @param pVM Pointer to the current VM (if any).
4907	* @param paArgs Pointer to (readonly) array of arguments.
4908	* @param cArgs Number of arguments in the array.
4909	*/
4910	static DECLCALLBACK(int) pgmR3CmdSyncAlways(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
4911	{
4912	/** @todo SMP support!! */
4913	PVMCPU pVCpu = &pVM->aCpus[0];
4914
4915	/*
4916	* Validate input.
4917	*/
4918	if (!pVM)
4919	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
4920
4921	/*
4922	* Force page directory sync.
4923	*/
4924	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS)
4925	{
4926	ASMAtomicAndU32(&pVCpu->pgm.s.fSyncFlags, ~PGM_SYNC_ALWAYS);
4927	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Disabled permanent forced page directory syncing.\n");
4928	}
4929	else
4930	{
4931	ASMAtomicOrU32(&pVCpu->pgm.s.fSyncFlags, PGM_SYNC_ALWAYS);
4932	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
4933	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Enabled permanent forced page directory syncing.\n");
4934	}
4935	}
4936
4937	#endif /* VBOX_WITH_DEBUGGER */
4938
4939	/**
4940	* pvUser argument of the pgmR3CheckIntegrity*Node callbacks.
4941	*/
4942	typedef struct PGMCHECKINTARGS
4943	{
4944	bool fLeftToRight; /*< true: left-to-right; false: right-to-left. /
4945	PPGMPHYSHANDLER pPrevPhys;
4946	PPGMVIRTHANDLER pPrevVirt;
4947	PPGMPHYS2VIRTHANDLER pPrevPhys2Virt;
4948	PVM pVM;
4949	} PGMCHECKINTARGS, *PPGMCHECKINTARGS;
4950
4951	/**
4952	* Validate a node in the physical handler tree.
4953	*
4954	* @returns 0 on if ok, other wise 1.
4955	* @param pNode The handler node.
4956	* @param pvUser pVM.
4957	*/
4958	static DECLCALLBACK(int) pgmR3CheckIntegrityPhysHandlerNode(PAVLROGCPHYSNODECORE pNode, void *pvUser)
4959	{
4960	PPGMCHECKINTARGS pArgs = (PPGMCHECKINTARGS)pvUser;
4961	PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)pNode;
4962	AssertReleaseReturn(!((uintptr_t)pCur & 7), 1);
4963	AssertReleaseMsg(pCur->Core.Key <= pCur->Core.KeyLast,("pCur=%p %RGp-%RGp %s\n", pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
4964	AssertReleaseMsg( !pArgs->pPrevPhys
4965	\|\| (pArgs->fLeftToRight ? pArgs->pPrevPhys->Core.KeyLast < pCur->Core.Key : pArgs->pPrevPhys->Core.KeyLast > pCur->Core.Key),
4966	("pPrevPhys=%p %RGp-%RGp %s\n"
4967	" pCur=%p %RGp-%RGp %s\n",
4968	pArgs->pPrevPhys, pArgs->pPrevPhys->Core.Key, pArgs->pPrevPhys->Core.KeyLast, pArgs->pPrevPhys->pszDesc,
4969	pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
4970	pArgs->pPrevPhys = pCur;
4971	return 0;
4972	}
4973
4974
4975	/**
4976	* Validate a node in the virtual handler tree.
4977	*
4978	* @returns 0 on if ok, other wise 1.
4979	* @param pNode The handler node.
4980	* @param pvUser pVM.
4981	*/
4982	static DECLCALLBACK(int) pgmR3CheckIntegrityVirtHandlerNode(PAVLROGCPTRNODECORE pNode, void *pvUser)
4983	{
4984	PPGMCHECKINTARGS pArgs = (PPGMCHECKINTARGS)pvUser;
4985	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)pNode;
4986	AssertReleaseReturn(!((uintptr_t)pCur & 7), 1);
4987	AssertReleaseMsg(pCur->Core.Key <= pCur->Core.KeyLast,("pCur=%p %RGv-%RGv %s\n", pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
4988	AssertReleaseMsg( !pArgs->pPrevVirt
4989	\|\| (pArgs->fLeftToRight ? pArgs->pPrevVirt->Core.KeyLast < pCur->Core.Key : pArgs->pPrevVirt->Core.KeyLast > pCur->Core.Key),
4990	("pPrevVirt=%p %RGv-%RGv %s\n"
4991	" pCur=%p %RGv-%RGv %s\n",
4992	pArgs->pPrevVirt, pArgs->pPrevVirt->Core.Key, pArgs->pPrevVirt->Core.KeyLast, pArgs->pPrevVirt->pszDesc,
4993	pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
4994	for (unsigned iPage = 0; iPage < pCur->cPages; iPage++)
4995	{
4996	AssertReleaseMsg(pCur->aPhysToVirt[iPage].offVirtHandler == -RT_OFFSETOF(PGMVIRTHANDLER, aPhysToVirt[iPage]),
4997	("pCur=%p %RGv-%RGv %s\n"
4998	"iPage=%d offVirtHandle=%#x expected %#x\n",
4999	pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc,
5000	iPage, pCur->aPhysToVirt[iPage].offVirtHandler, -RT_OFFSETOF(PGMVIRTHANDLER, aPhysToVirt[iPage])));
5001	}
5002	pArgs->pPrevVirt = pCur;
5003	return 0;
5004	}
5005
5006
5007	/**
5008	* Validate a node in the virtual handler tree.
5009	*
5010	* @returns 0 on if ok, other wise 1.
5011	* @param pNode The handler node.
5012	* @param pvUser pVM.
5013	*/
5014	static DECLCALLBACK(int) pgmR3CheckIntegrityPhysToVirtHandlerNode(PAVLROGCPHYSNODECORE pNode, void *pvUser)
5015	{
5016	PPGMCHECKINTARGS pArgs = (PPGMCHECKINTARGS)pvUser;
5017	PPGMPHYS2VIRTHANDLER pCur = (PPGMPHYS2VIRTHANDLER)pNode;
5018	AssertReleaseMsgReturn(!((uintptr_t)pCur & 3), ("\n"), 1);
5019	AssertReleaseMsgReturn(!(pCur->offVirtHandler & 3), ("\n"), 1);
5020	AssertReleaseMsg(pCur->Core.Key <= pCur->Core.KeyLast,("pCur=%p %RGp-%RGp\n", pCur, pCur->Core.Key, pCur->Core.KeyLast));
5021	AssertReleaseMsg( !pArgs->pPrevPhys2Virt
5022	\|\| (pArgs->fLeftToRight ? pArgs->pPrevPhys2Virt->Core.KeyLast < pCur->Core.Key : pArgs->pPrevPhys2Virt->Core.KeyLast > pCur->Core.Key),
5023	("pPrevPhys2Virt=%p %RGp-%RGp\n"
5024	" pCur=%p %RGp-%RGp\n",
5025	pArgs->pPrevPhys2Virt, pArgs->pPrevPhys2Virt->Core.Key, pArgs->pPrevPhys2Virt->Core.KeyLast,
5026	pCur, pCur->Core.Key, pCur->Core.KeyLast));
5027	AssertReleaseMsg( !pArgs->pPrevPhys2Virt
5028	\|\| (pArgs->fLeftToRight ? pArgs->pPrevPhys2Virt->Core.KeyLast < pCur->Core.Key : pArgs->pPrevPhys2Virt->Core.KeyLast > pCur->Core.Key),
5029	("pPrevPhys2Virt=%p %RGp-%RGp\n"
5030	" pCur=%p %RGp-%RGp\n",
5031	pArgs->pPrevPhys2Virt, pArgs->pPrevPhys2Virt->Core.Key, pArgs->pPrevPhys2Virt->Core.KeyLast,
5032	pCur, pCur->Core.Key, pCur->Core.KeyLast));
5033	AssertReleaseMsg((pCur->offNextAlias & (PGMPHYS2VIRTHANDLER_IN_TREE \| PGMPHYS2VIRTHANDLER_IS_HEAD)) == (PGMPHYS2VIRTHANDLER_IN_TREE \| PGMPHYS2VIRTHANDLER_IS_HEAD),
5034	("pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
5035	pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias));
5036	if (pCur->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK)
5037	{
5038	PPGMPHYS2VIRTHANDLER pCur2 = pCur;
5039	for (;;)
5040	{
5041	pCur2 = (PPGMPHYS2VIRTHANDLER)((intptr_t)pCur + (pCur->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
5042	AssertReleaseMsg(pCur2 != pCur,
5043	(" pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
5044	pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias));
5045	AssertReleaseMsg((pCur2->offNextAlias & (PGMPHYS2VIRTHANDLER_IN_TREE \| PGMPHYS2VIRTHANDLER_IS_HEAD)) == PGMPHYS2VIRTHANDLER_IN_TREE,
5046	(" pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
5047	"pCur2=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
5048	pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias,
5049	pCur2, pCur2->Core.Key, pCur2->Core.KeyLast, pCur2->offVirtHandler, pCur2->offNextAlias));
5050	AssertReleaseMsg((pCur2->Core.Key ^ pCur->Core.Key) < PAGE_SIZE,
5051	(" pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
5052	"pCur2=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
5053	pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias,
5054	pCur2, pCur2->Core.Key, pCur2->Core.KeyLast, pCur2->offVirtHandler, pCur2->offNextAlias));
5055	AssertReleaseMsg((pCur2->Core.KeyLast ^ pCur->Core.KeyLast) < PAGE_SIZE,
5056	(" pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
5057	"pCur2=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
5058	pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias,
5059	pCur2, pCur2->Core.Key, pCur2->Core.KeyLast, pCur2->offVirtHandler, pCur2->offNextAlias));
5060	if (!(pCur2->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK))
5061	break;
5062	}
5063	}
5064
5065	pArgs->pPrevPhys2Virt = pCur;
5066	return 0;
5067	}
5068
5069
5070	/**
5071	* Perform an integrity check on the PGM component.
5072	*
5073	* @returns VINF_SUCCESS if everything is fine.
5074	* @returns VBox error status after asserting on integrity breach.
5075	* @param pVM The VM handle.
5076	*/
5077	VMMR3DECL(int) PGMR3CheckIntegrity(PVM pVM)
5078	{
5079	AssertReleaseReturn(pVM->pgm.s.offVM, VERR_INTERNAL_ERROR);
5080
5081	/*
5082	* Check the trees.
5083	*/
5084	int cErrors = 0;
5085	const static PGMCHECKINTARGS s_LeftToRight = { true, NULL, NULL, NULL, pVM };
5086	const static PGMCHECKINTARGS s_RightToLeft = { false, NULL, NULL, NULL, pVM };
5087	PGMCHECKINTARGS Args = s_LeftToRight;
5088	cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers, true, pgmR3CheckIntegrityPhysHandlerNode, &Args);
5089	Args = s_RightToLeft;
5090	cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers, false, pgmR3CheckIntegrityPhysHandlerNode, &Args);
5091	Args = s_LeftToRight;
5092	cErrors += RTAvlroGCPtrDoWithAll( &pVM->pgm.s.pTreesR3->VirtHandlers, true, pgmR3CheckIntegrityVirtHandlerNode, &Args);
5093	Args = s_RightToLeft;
5094	cErrors += RTAvlroGCPtrDoWithAll( &pVM->pgm.s.pTreesR3->VirtHandlers, false, pgmR3CheckIntegrityVirtHandlerNode, &Args);
5095	Args = s_LeftToRight;
5096	cErrors += RTAvlroGCPtrDoWithAll( &pVM->pgm.s.pTreesR3->HyperVirtHandlers, true, pgmR3CheckIntegrityVirtHandlerNode, &Args);
5097	Args = s_RightToLeft;
5098	cErrors += RTAvlroGCPtrDoWithAll( &pVM->pgm.s.pTreesR3->HyperVirtHandlers, false, pgmR3CheckIntegrityVirtHandlerNode, &Args);
5099	Args = s_LeftToRight;
5100	cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysToVirtHandlers, true, pgmR3CheckIntegrityPhysToVirtHandlerNode, &Args);
5101	Args = s_RightToLeft;
5102	cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysToVirtHandlers, false, pgmR3CheckIntegrityPhysToVirtHandlerNode, &Args);
5103
5104	return !cErrors ? VINF_SUCCESS : VERR_INTERNAL_ERROR;
5105	}
5106
5107

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/PGM.cpp@ 21134

Download in other formats: