Changeset 63551 in vbox for trunk/src/VBox/Additions/linux/drm

Timestamp:

Aug 16, 2016 1:02:19 PM (8 years ago)

Author:

vboxsync

Message:

bugref:8087: Additions/x11: support non-root X server: the modesetting driver with X.Org Server prior to version 1.18.4 had a bug which caused it do disable dirty updates if it submitted more rectangles at once than the kernel wished to handle. Work around that by hiding the EINVAL error code that it looks for. Time-limited to pre-4.7 kernels, which should be enough.

File:

• trunk/src/VBox/Additions/linux/drm/vbox_drv.c (modified) (2 diffs)

trunk/src/VBox/Additions/linux/drm/vbox_drv.c

@@ -195 +195 @@
 };
 
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)
+/* This works around a bug in X servers prior to 1.18.4, which sometimes
+ * submit more dirty rectangles than the kernel is willing to handle and
+ * then disable dirty rectangle handling altogether when they see the
+ * EINVAL error.  I do not want the code to hang around forever, which is
+ * why I am limiting it to certain kernel versions.  We can increase the
+ * limit if some distributions uses old X servers with new kernels. */
+long vbox_ioctl(struct file *filp,
+                unsigned int cmd, unsigned long arg)
+{
+    long rc = drm_ioctl(filp, cmd, arg);
+    if (cmd == DRM_IOCTL_MODE_DIRTYFB && rc == -EINVAL)
+        return -EOVERFLOW;
+    return rc;
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0) */
+
 static const struct file_operations vbox_fops =
 {
@@ -200 +217 @@
     .open = drm_open,
     .release = drm_release,
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)
+    .unlocked_ioctl = vbox_ioctl,
+#else
     .unlocked_ioctl = drm_ioctl,
+#endif
     .mmap = vbox_mmap,
     .poll = drm_poll,