drm/i915: introduce page_size members

In preparation for supporting huge gtt pages for the ppgtt, we introduce
page size members for gem objects.  We fill in the page sizes by
scanning the sg table.

v2: pass the sg_mask to set_pages

v3: calculate the sg_mask inline with populating the sg_table where
possible, and pass to set_pages along with the pages.

v4: bunch of improvements from Joonas

v5: fix num_pages blunder
    introduce i915_sg_page_sizes helper

v6: prefer GEM_BUG_ON(sizes == 0)

Signed-off-by: Matthew Auld <matthew.auld@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Daniel Vetter <daniel@ffwll.ch>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171006145041.21673-7-matthew.auld@intel.com
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171006221833.32439-6-chris@chris-wilson.co.uk

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 3d4dee817381350b4cd0038003ff2f9ba4c2d7aa..799a90abd81f267a9588ad0c022326b108f4b8d7 100644 (file)
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -2872,6 +2872,21 @@ static inline struct scatterlist *__sg_next(struct scatterlist *sg)
             (((__iter).curr += PAGE_SIZE) >= (__iter).max) ?           \
             (__iter) = __sgt_iter(__sg_next((__iter).sgp), false), 0 : 0)
 
+static inline unsigned int i915_sg_page_sizes(struct scatterlist *sg)
+{
+       unsigned int page_sizes;
+
+       page_sizes = 0;
+       while (sg) {
+               GEM_BUG_ON(sg->offset);
+               GEM_BUG_ON(!IS_ALIGNED(sg->length, PAGE_SIZE));
+               page_sizes |= sg->length;
+               sg = __sg_next(sg);
+       }
+
+       return page_sizes;
+}
+
 static inline unsigned int i915_sg_segment_size(void)
 {
        unsigned int size = swiotlb_max_segment();
@@ -3101,6 +3116,10 @@ intel_info(const struct drm_i915_private *dev_priv)
 #define USES_PPGTT(dev_priv)           (i915_modparams.enable_ppgtt)
 #define USES_FULL_PPGTT(dev_priv)      (i915_modparams.enable_ppgtt >= 2)
 #define USES_FULL_48BIT_PPGTT(dev_priv)        (i915_modparams.enable_ppgtt == 3)
+#define HAS_PAGE_SIZES(dev_priv, sizes) ({ \
+       GEM_BUG_ON((sizes) == 0); \
+       ((sizes) & ~(dev_priv)->info.page_sizes) == 0; \
+})
 
 #define HAS_OVERLAY(dev_priv)           ((dev_priv)->info.has_overlay)
 #define OVERLAY_NEEDS_PHYSICAL(dev_priv) \
@@ -3517,7 +3536,8 @@ i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
                                unsigned long n);
 
 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
-                                struct sg_table *pages);
+                                struct sg_table *pages,
+                                unsigned int sg_mask);
 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
 
 static inline int __must_check

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 42f2ca1e136b4ceb72fe885b6952df931ebc9f4a..34398696824c65bd84df0dc7008159426035239c 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -228,7 +228,7 @@ static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 
        obj->phys_handle = phys;
 
-       __i915_gem_object_set_pages(obj, st);
+       __i915_gem_object_set_pages(obj, st, sg->length);
 
        return 0;
 
@@ -2266,6 +2266,8 @@ void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj,
        if (!IS_ERR(pages))
                obj->ops->put_pages(obj, pages);
 
+       obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
+
 unlock:
        mutex_unlock(&obj->mm.lock);
 }
@@ -2308,6 +2310,7 @@ static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
        struct page *page;
        unsigned long last_pfn = 0;     /* suppress gcc warning */
        unsigned int max_segment = i915_sg_segment_size();
+       unsigned int sg_mask;
        gfp_t noreclaim;
        int ret;
 
@@ -2339,6 +2342,7 @@ rebuild_st:
 
        sg = st->sgl;
        st->nents = 0;
+       sg_mask = 0;
        for (i = 0; i < page_count; i++) {
                const unsigned int shrink[] = {
                        I915_SHRINK_BOUND | I915_SHRINK_UNBOUND | I915_SHRINK_PURGEABLE,
@@ -2391,8 +2395,10 @@ rebuild_st:
                if (!i ||
                    sg->length >= max_segment ||
                    page_to_pfn(page) != last_pfn + 1) {
-                       if (i)
+                       if (i) {
+                               sg_mask |= sg->length;
                                sg = sg_next(sg);
+                       }
                        st->nents++;
                        sg_set_page(sg, page, PAGE_SIZE, 0);
                } else {
@@ -2403,8 +2409,10 @@ rebuild_st:
                /* Check that the i965g/gm workaround works. */
                WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
        }
-       if (sg) /* loop terminated early; short sg table */
+       if (sg) { /* loop terminated early; short sg table */
+               sg_mask |= sg->length;
                sg_mark_end(sg);
+       }
 
        /* Trim unused sg entries to avoid wasting memory. */
        i915_sg_trim(st);
@@ -2433,7 +2441,7 @@ rebuild_st:
        if (i915_gem_object_needs_bit17_swizzle(obj))
                i915_gem_object_do_bit_17_swizzle(obj, st);
 
-       __i915_gem_object_set_pages(obj, st);
+       __i915_gem_object_set_pages(obj, st, sg_mask);
 
        return 0;
 
@@ -2460,8 +2468,13 @@ err_pages:
 }
 
 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
-                                struct sg_table *pages)
+                                struct sg_table *pages,
+                                unsigned int sg_mask)
 {
+       struct drm_i915_private *i915 = to_i915(obj->base.dev);
+       unsigned long supported = INTEL_INFO(i915)->page_sizes;
+       int i;
+
        lockdep_assert_held(&obj->mm.lock);
 
        obj->mm.get_page.sg_pos = pages->sgl;
@@ -2475,6 +2488,25 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
                __i915_gem_object_pin_pages(obj);
                obj->mm.quirked = true;
        }
+
+       GEM_BUG_ON(!sg_mask);
+       obj->mm.page_sizes.phys = sg_mask;
+
+       /*
+        * Calculate the supported page-sizes which fit into the given sg_mask.
+        * This will give us the page-sizes which we may be able to use
+        * opportunistically when later inserting into the GTT. For example if
+        * phys=2G, then in theory we should be able to use 1G, 2M, 64K or 4K
+        * pages, although in practice this will depend on a number of other
+        * factors.
+        */
+       obj->mm.page_sizes.sg = 0;
+       for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
+               if (obj->mm.page_sizes.phys & ~0u << i)
+                       obj->mm.page_sizes.sg |= BIT(i);
+       }
+
+       GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
 }
 
 static int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)

diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/i915_gem_dmabuf.c
index 4c4dc85159fb4b31e7b62743ceb2a19903ae82f9..e542a9d8007770346a7bac0a29981a9edc809278 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem_dmabuf.c
+++ b/drivers/gpu/drm/i915/i915_gem_dmabuf.c
@@ -259,13 +259,16 @@ struct dma_buf *i915_gem_prime_export(struct drm_device *dev,
 static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)
 {
        struct sg_table *pages;
+       unsigned int sg_mask;
 
        pages = dma_buf_map_attachment(obj->base.import_attach,
                                       DMA_BIDIRECTIONAL);
        if (IS_ERR(pages))
                return PTR_ERR(pages);
 
-       __i915_gem_object_set_pages(obj, pages);
+       sg_mask = i915_sg_page_sizes(pages->sgl);
+
+       __i915_gem_object_set_pages(obj, pages, sg_mask);
 
        return 0;
 }

diff --git a/drivers/gpu/drm/i915/i915_gem_internal.c b/drivers/gpu/drm/i915/i915_gem_internal.c
index f59764da42544a413370a0be93c0d0fdda5df673..bdc23c4c8783081346a619ae745f27641f168ea4 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem_internal.c
+++ b/drivers/gpu/drm/i915/i915_gem_internal.c
@@ -49,6 +49,7 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
        struct drm_i915_private *i915 = to_i915(obj->base.dev);
        struct sg_table *st;
        struct scatterlist *sg;
+       unsigned int sg_mask;
        unsigned int npages;
        int max_order;
        gfp_t gfp;
@@ -87,6 +88,7 @@ create_st:
 
        sg = st->sgl;
        st->nents = 0;
+       sg_mask = 0;
 
        do {
                int order = min(fls(npages) - 1, max_order);
@@ -104,6 +106,7 @@ create_st:
                } while (1);
 
                sg_set_page(sg, page, PAGE_SIZE << order, 0);
+               sg_mask |= PAGE_SIZE << order;
                st->nents++;
 
                npages -= 1 << order;
@@ -132,7 +135,7 @@ create_st:
         */
        obj->mm.madv = I915_MADV_DONTNEED;
 
-       __i915_gem_object_set_pages(obj, st);
+       __i915_gem_object_set_pages(obj, st, sg_mask);
 
        return 0;
 

diff --git a/drivers/gpu/drm/i915/i915_gem_object.h b/drivers/gpu/drm/i915/i915_gem_object.h
index 036e847b27f0a2c10b57d479bb9e55f329266e64..110672952a1ca61fa3346e108ea9d1ad6d03c170 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/i915_gem_object.h
@@ -169,6 +169,23 @@ struct drm_i915_gem_object {
                struct sg_table *pages;
                void *mapping;
 
+               struct i915_page_sizes {
+                       /**
+                        * The sg mask of the pages sg_table. i.e the mask of
+                        * of the lengths for each sg entry.
+                        */
+                       unsigned int phys;
+
+                       /**
+                        * The gtt page sizes we are allowed to use given the
+                        * sg mask and the supported page sizes. This will
+                        * express the smallest unit we can use for the whole
+                        * object, as well as the larger sizes we may be able
+                        * to use opportunistically.
+                        */
+                       unsigned int sg;
+               } page_sizes;
+
                struct i915_gem_object_page_iter {
                        struct scatterlist *sg_pos;
                        unsigned int sg_idx; /* in pages, but 32bit eek! */

diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index 537ecb224db0ee4c874205d6f728d356da7a8784..54fd4cfa9d071036224186c45ae02558742ee013 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -548,7 +548,7 @@ static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)
        if (IS_ERR(pages))
                return PTR_ERR(pages);
 
-       __i915_gem_object_set_pages(obj, pages);
+       __i915_gem_object_set_pages(obj, pages, obj->stolen->size);
 
        return 0;
 }

diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c
index 70ad7489827dfdf23a0b8c212bd4ef6c53eb69a6..41e16e19c3f37521ed3cb690c07ec17184b88b70 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/i915_gem_userptr.c
@@ -405,6 +405,7 @@ __i915_gem_userptr_alloc_pages(struct drm_i915_gem_object *obj,
 {
        unsigned int max_segment = i915_sg_segment_size();
        struct sg_table *st;
+       unsigned int sg_mask;
        int ret;
 
        st = kmalloc(sizeof(*st), GFP_KERNEL);
@@ -434,7 +435,9 @@ alloc_table:
                return ERR_PTR(ret);
        }
 
-       __i915_gem_object_set_pages(obj, st);
+       sg_mask = i915_sg_page_sizes(st->sgl);
+
+       __i915_gem_object_set_pages(obj, st, sg_mask);
 
        return st;
 }

diff --git a/drivers/gpu/drm/i915/selftests/huge_gem_object.c b/drivers/gpu/drm/i915/selftests/huge_gem_object.c
index 41c15f3aa467a4f25822cfc13c98a0239d123251..a2632df3917353b246a1316f3322811476c24418 100644 (file)
--- a/drivers/gpu/drm/i915/selftests/huge_gem_object.c
+++ b/drivers/gpu/drm/i915/selftests/huge_gem_object.c
@@ -80,7 +80,7 @@ static int huge_get_pages(struct drm_i915_gem_object *obj)
        if (i915_gem_gtt_prepare_pages(obj, pages))
                goto err;
 
-       __i915_gem_object_set_pages(obj, pages);
+       __i915_gem_object_set_pages(obj, pages, PAGE_SIZE);
 
        return 0;
 

diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
index aa1db375d59ab59940ca4d5bf568c4350eeba560..883bc19e3aafef2b31bf78ab04c5721c8a98e155 100644 (file)
--- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
@@ -45,6 +45,7 @@ static int fake_get_pages(struct drm_i915_gem_object *obj)
 #define PFN_BIAS 0x1000
        struct sg_table *pages;
        struct scatterlist *sg;
+       unsigned int sg_mask;
        typeof(obj->base.size) rem;
 
        pages = kmalloc(sizeof(*pages), GFP);
@@ -57,6 +58,7 @@ static int fake_get_pages(struct drm_i915_gem_object *obj)
                return -ENOMEM;
        }
 
+       sg_mask = 0;
        rem = obj->base.size;
        for (sg = pages->sgl; sg; sg = sg_next(sg)) {
                unsigned long len = min_t(typeof(rem), rem, BIT(31));
@@ -65,6 +67,7 @@ static int fake_get_pages(struct drm_i915_gem_object *obj)
                sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0);
                sg_dma_address(sg) = page_to_phys(sg_page(sg));
                sg_dma_len(sg) = len;
+               sg_mask |= len;
 
                rem -= len;
        }
@@ -72,7 +75,7 @@ static int fake_get_pages(struct drm_i915_gem_object *obj)
 
        obj->mm.madv = I915_MADV_DONTNEED;
 
-       __i915_gem_object_set_pages(obj, pages);
+       __i915_gem_object_set_pages(obj, pages, sg_mask);
 
        return 0;
 #undef GFP