Changeset 62550 in vbox

Timestamp:

Jul 25, 2016 6:32:58 PM (9 years ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

109119

Message:

bugref:8087: Additions/x11: support non-root X server: do not map all of the guest VRAM when we only need the guest/host communication areas. This could fail on 32-bit guests (and possibly 64-bit too) with large VRAM sizes, making the guest unusable.

Location:

trunk/src/VBox/Additions/linux/drm

Files:

• vbox_drv.h (modified) (3 diffs)
• vbox_fb.c (modified) (1 diff)
• vbox_irq.c (modified) (1 diff)
• vbox_main.c (modified) (9 diffs)
• vbox_mode.c (modified) (1 diff)
• vbox_ttm.c (modified) (1 diff)

trunk/src/VBox/Additions/linux/drm/vbox_drv.h

@@ -88 +88 @@
 #define CURSOR_DATA_SIZE CURSOR_PIXEL_COUNT * 4 + CURSOR_PIXEL_COUNT / 8
 
+#define VBOX_MAX_SCREENS  32
+
 struct vbox_fbdev;
 
@@ -93 +95 @@
     struct drm_device *dev;
 
-    uint8_t __iomem *vram;
+    uint8_t __iomem *mapped_vram;
     HGSMIGUESTCOMMANDCONTEXT submit_info;
     struct VBVABUFFERCONTEXT *vbva_info;
@@ -102 +104 @@
     uint32_t full_vram_size;
     /** Amount of available VRAM, not including space used for buffers. */
-    uint32_t vram_size;
+    uint32_t available_vram_size;
+    /** Offset of host communication area in mapped VRAM. */
+    uint32_t vram_host_offset;
     /** Offset to the host flags in the VRAM. */
     uint32_t host_flags_offset;

trunk/src/VBox/Additions/linux/drm/vbox_fb.c

@@ -498 +498 @@
                 vbox->fbdev->helper.fbdev->apertures->ranges[0].base +
                 gpu_addr;
-        vbox->fbdev->helper.fbdev->fix.smem_len = vbox->vram_size - gpu_addr;
-}
+        vbox->fbdev->helper.fbdev->fix.smem_len = vbox->available_vram_size - gpu_addr;
+}

trunk/src/VBox/Additions/linux/drm/vbox_irq.c

@@ -57 +57 @@
 static uint32_t vbox_get_flags(struct vbox_private *vbox)
 {
-    return (uint32_t)readl(vbox->vram + vbox->host_flags_offset);
+    return (uint32_t)readl(vbox->mapped_vram + vbox->host_flags_offset);
 }
 

trunk/src/VBox/Additions/linux/drm/vbox_main.c

@@ -75 +75 @@
         if (vbox->vbva_info[i].pVBVA == NULL) {
             LogFunc(("vboxvideo: enabling VBVA.\n"));
-            vbva = (struct VBVABUFFER *) (  ((uint8_t *)vbox->vram)
-                                           + vbox->vram_size
+            vbva = (struct VBVABUFFER *) (  ((uint8_t *)vbox->mapped_vram)
+                                           + vbox->vram_host_offset
                                            + i * VBVA_MIN_BUFFER_SIZE);
             if (!VBoxVBVAEnable(&vbox->vbva_info[i], &vbox->submit_info, vbva, i))
@@ -255 +255 @@
     }
     /* Take a command buffer for each screen from the end of usable VRAM. */
-    vbox->vram_size -= vbox->num_crtcs * VBVA_MIN_BUFFER_SIZE;
+    vbox->vram_host_offset = (VBOX_MAX_SCREENS - vbox->num_crtcs) * VBVA_MIN_BUFFER_SIZE;
+    vbox->available_vram_size -= vbox->num_crtcs * VBVA_MIN_BUFFER_SIZE;
     for (i = 0; i < vbox->num_crtcs; ++i)
         VBoxVBVASetupBufferContext(&vbox->vbva_info[i],
-                                   vbox->vram_size + i * VBVA_MIN_BUFFER_SIZE,
+                                   vbox->available_vram_size + i * VBVA_MIN_BUFFER_SIZE,
                                    VBVA_MIN_BUFFER_SIZE);
-    LogFunc(("vboxvideo: %d: vbox->vbva_info=%p, vbox->vram_size=%u\n",
-             __LINE__, vbox->vbva_info, (unsigned)vbox->vram_size));
+    LogFunc(("vboxvideo: %d: vbox->vbva_info=%p, vbox->available_vram_size=%u\n",
+             __LINE__, vbox->vbva_info, (unsigned)vbox->available_vram_size));
     return 0;
 }
@@ -301 +302 @@
 }
 
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
+# define pci_iomap_range(dev, bar, offset, maxlen) \
+    ioremap(pci_resource_start(dev, bar) + offset, maxlen)
+#endif
 
 /** Set up our heaps and data exchange buffers in VRAM before handing the rest
@@ -306 +311 @@
 static int vbox_hw_init(struct vbox_private *vbox)
 {
-    uint32_t base_offset, guest_heap_offset, guest_heap_size, host_flags_offset;
+    uint32_t base_offset, map_start, guest_heap_offset, guest_heap_size, host_flags_offset;
     void *guest_heap;
 
@@ -314 +319 @@
     VBoxHGSMIGetBaseMappingInfo(vbox->full_vram_size, &base_offset, NULL,
                                 &guest_heap_offset, &guest_heap_size, &host_flags_offset);
-    guest_heap =   ((uint8_t *)vbox->vram) + base_offset + guest_heap_offset;
-    vbox->host_flags_offset = base_offset + host_flags_offset;
+    map_start = (uint32_t)max((int)base_offset
+                              - VBOX_MAX_SCREENS * VBVA_MIN_BUFFER_SIZE, 0);
+    vbox->mapped_vram = pci_iomap_range(vbox->dev->pdev, 0, map_start,
+                                        vbox->full_vram_size - map_start);
+    if (!vbox->mapped_vram)
+        return -ENOMEM;
+    guest_heap = ((uint8_t *)vbox->mapped_vram) + base_offset - map_start
+                   + guest_heap_offset;
+    vbox->host_flags_offset = base_offset - map_start + host_flags_offset;
     if (RT_FAILURE(VBoxHGSMISetupGuestContext(&vbox->submit_info, guest_heap,
                                               guest_heap_size,
@@ -322 +334 @@
         return -ENOMEM;
     /* Reduce available VRAM size to reflect the guest heap. */
-    vbox->vram_size = base_offset;
+    vbox->available_vram_size = base_offset;
     /* Linux drm represents monitors as a 32-bit array. */
-    vbox->num_crtcs = RT_MIN(VBoxHGSMIGetMonitorCount(&vbox->submit_info), 32);
+    vbox->num_crtcs = min(VBoxHGSMIGetMonitorCount(&vbox->submit_info),
+                          (uint32_t)VBOX_MAX_SCREENS);
     if (!have_hgsmi_mode_hints(vbox))
         return -ENOTSUPP;
@@ -357 +370 @@
 
     mutex_init(&vbox->hw_mutex);
-    /* I hope this won't interfere with the memory manager. */
-    vbox->vram = pci_iomap(dev->pdev, 0, 0);
-    if (!vbox->vram) {
-        ret = -EIO;
-        goto out_free;
-    }
 
     ret = vbox_hw_init(vbox);
@@ -392 +399 @@
     if (ret)
         goto out_free;
-    LogFunc(("vboxvideo: %d: vbox=%p, vbox->vram=%p, vbox->full_vram_size=%u\n",
-             __LINE__, vbox, vbox->vram, (unsigned)vbox->full_vram_size));
+    LogFunc(("vboxvideo: %d: vbox=%p, vbox->mapped_vram=%p, vbox->full_vram_size=%u\n",
+             __LINE__, vbox, vbox->mapped_vram, (unsigned)vbox->full_vram_size));
     return 0;
 out_free:
@@ -414 +421 @@
     vbox_hw_fini(vbox);
     vbox_mm_fini(vbox);
-    if (vbox->vram)
-        pci_iounmap(dev->pdev, vbox->vram);
+    if (vbox->mapped_vram)
+        pci_iounmap(dev->pdev, vbox->mapped_vram);
     kfree(vbox);
     dev->dev_private = NULL;

trunk/src/VBox/Additions/linux/drm/vbox_mode.c

@@ -133 +133 @@
         pInfo->u32ViewIndex = vbox_crtc->crtc_id;
         pInfo->u32ViewOffset = vbox_crtc->fb_offset;
-        pInfo->u32ViewSize =   vbox->vram_size - vbox_crtc->fb_offset
+        pInfo->u32ViewSize =   vbox->available_vram_size - vbox_crtc->fb_offset
                              + vbox_crtc->crtc_id * VBVA_MIN_BUFFER_SIZE;
-        pInfo->u32MaxScreenSize = vbox->vram_size - vbox_crtc->fb_offset;
+        pInfo->u32MaxScreenSize = vbox->available_vram_size - vbox_crtc->fb_offset;
         VBoxHGSMIBufferSubmit(&vbox->submit_info, p);
         VBoxHGSMIBufferFree(&vbox->submit_info, p);

trunk/src/VBox/Additions/linux/drm/vbox_ttm.c

@@ -301 +301 @@
 
     ret = ttm_bo_init_mm(bdev, TTM_PL_VRAM,
-                 vbox->vram_size >> PAGE_SHIFT);
+                 vbox->available_vram_size >> PAGE_SHIFT);
     if (ret) {
         DRM_ERROR("Failed ttm VRAM init: %d\n", ret);