drm/vmwgfx: limit screen size to stdu_max during check_modeset

For STDU individual screen target size is limited by
SVGA_REG_SCREENTARGET_MAX_WIDTH/HEIGHT registers so add that limit
during atomic check_modeset.

An additional limit is placed in the update_layout ioctl to avoid
requesting layouts that current user-space typically can't support.
Also modified the comments to reflect current limitation on topology.

Signed-off-by: Deepak Rawat <drawat@vmware.com>
Reviewed-by: Sinclair Yeh <syeh@vmware.com>
Reviewed-by: Thomas Hellstrom <thellstrom@vmware.com>
Signed-off-by: Thomas Hellstrom <thellstrom@vmware.com>

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 636b962849c8275c6267be41fb67e83ff32ba362..12a41b0391678e664907cdf964d53be96cb841ce 100644 (file)
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -1512,21 +1512,19 @@ static int vmw_kms_check_display_memory(struct drm_device *dev,
                                        struct drm_rect *rects)
 {
        struct vmw_private *dev_priv = vmw_priv(dev);
-       struct drm_mode_config *mode_config = &dev->mode_config;
        struct drm_rect bounding_box = {0};
        u64 total_pixels = 0, pixel_mem, bb_mem;
        int i;
 
        for (i = 0; i < num_rects; i++) {
                /*
-                * Currently this check is limiting the topology within max
-                * texture/screentarget size. This should change in future when
-                * user-space support multiple fb with topology.
+                * For STDU only individual screen (screen target) is limited by
+                * SCREENTARGET_MAX_WIDTH/HEIGHT registers.
                 */
-               if (rects[i].x1 < 0 ||  rects[i].y1 < 0 ||
-                   rects[i].x2 > mode_config->max_width ||
-                   rects[i].y2 > mode_config->max_height) {
-                       DRM_ERROR("Invalid GUI layout.\n");
+               if (dev_priv->active_display_unit == vmw_du_screen_target &&
+                   (drm_rect_width(&rects[i]) > dev_priv->stdu_max_width ||
+                    drm_rect_height(&rects[i]) > dev_priv->stdu_max_height)) {
+                       DRM_ERROR("Screen size not supported.\n");
                        return -EINVAL;
                }
 
@@ -2376,6 +2374,7 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
                                struct drm_file *file_priv)
 {
        struct vmw_private *dev_priv = vmw_priv(dev);
+       struct drm_mode_config *mode_config = &dev->mode_config;
        struct drm_vmw_update_layout_arg *arg =
                (struct drm_vmw_update_layout_arg *)data;
        void __user *user_rects;
@@ -2421,6 +2420,21 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
                drm_rects[i].y1 = curr_rect.y;
                drm_rects[i].x2 = curr_rect.x + curr_rect.w;
                drm_rects[i].y2 = curr_rect.y + curr_rect.h;
+
+               /*
+                * Currently this check is limiting the topology within
+                * mode_config->max (which actually is max texture size
+                * supported by virtual device). This limit is here to address
+                * window managers that create a big framebuffer for whole
+                * topology.
+                */
+               if (drm_rects[i].x1 < 0 ||  drm_rects[i].y1 < 0 ||
+                   drm_rects[i].x2 > mode_config->max_width ||
+                   drm_rects[i].y2 > mode_config->max_height) {
+                       DRM_ERROR("Invalid GUI layout.\n");
+                       ret = -EINVAL;
+                       goto out_free;
+               }
        }
 
        ret = vmw_kms_check_display_memory(dev, arg->num_outputs, drm_rects);