]> git.baikalelectronics.ru Git - kernel.git/commitdiff
drm/ttm: Add a generic TTM memcpy move for page-based iomem
authorThomas Hellström <thomas.hellstrom@linux.intel.com>
Wed, 2 Jun 2021 08:38:10 +0000 (10:38 +0200)
committerThomas Hellström <thomas.hellstrom@linux.intel.com>
Mon, 7 Jun 2021 14:07:08 +0000 (16:07 +0200)
The internal ttm_bo_util memcpy uses ioremap functionality, and while it
probably might be possible to use it for copying in- and out of
sglist represented io memory, using io_mem_reserve() / io_mem_free()
callbacks, that would cause problems with fault().
Instead, implement a method mapping page-by-page using kmap_local()
semantics. As an additional benefit we then avoid the occasional global
TLB flushes of ioremap() and consuming ioremap space, elimination of a
critical point of failure and with a slight change of semantics we could
also push the memcpy out async for testing and async driver development
purposes.

A special linear iomem iterator is introduced internally to mimic the
old ioremap behaviour for code-paths that can't immediately be ported
over. This adds to the code size and should be considered a temporary
solution.

Looking at the code we have a lot of checks for iomap tagged pointers.
Ideally we should extend the core memremap functions to also accept
uncached memory and kmap_local functionality. Then we could strip a
lot of code.

Cc: Christian König <christian.koenig@amd.com>
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Reviewed-by: Christian König <christian.koenig@amd.com>
Link: https://lore.kernel.org/r/20210602083818.241793-4-thomas.hellstrom@linux.intel.com
drivers/gpu/drm/ttm/ttm_bo_util.c
drivers/gpu/drm/ttm/ttm_module.c
drivers/gpu/drm/ttm/ttm_resource.c
drivers/gpu/drm/ttm/ttm_tt.c
include/drm/ttm/ttm_bo_driver.h
include/drm/ttm/ttm_caching.h
include/drm/ttm/ttm_kmap_iter.h [new file with mode: 0644]
include/drm/ttm/ttm_resource.h
include/drm/ttm/ttm_tt.h

index 1b326e70cb022ab7d011637e2924aded07b5ad23..0d1e417e62d61fc95d1b17fcb75136aae6839342 100644 (file)
@@ -72,188 +72,125 @@ void ttm_mem_io_free(struct ttm_device *bdev,
        mem->bus.addr = NULL;
 }
 
-static int ttm_resource_ioremap(struct ttm_device *bdev,
-                              struct ttm_resource *mem,
-                              void **virtual)
+/**
+ * ttm_move_memcpy - Helper to perform a memcpy ttm move operation.
+ * @bo: The struct ttm_buffer_object.
+ * @new_mem: The struct ttm_resource we're moving to (copy destination).
+ * @new_iter: A struct ttm_kmap_iter representing the destination resource.
+ * @src_iter: A struct ttm_kmap_iter representing the source resource.
+ *
+ * This function is intended to be able to move out async under a
+ * dma-fence if desired.
+ */
+void ttm_move_memcpy(struct ttm_buffer_object *bo,
+                    u32 num_pages,
+                    struct ttm_kmap_iter *dst_iter,
+                    struct ttm_kmap_iter *src_iter)
 {
-       int ret;
-       void *addr;
-
-       *virtual = NULL;
-       ret = ttm_mem_io_reserve(bdev, mem);
-       if (ret || !mem->bus.is_iomem)
-               return ret;
+       const struct ttm_kmap_iter_ops *dst_ops = dst_iter->ops;
+       const struct ttm_kmap_iter_ops *src_ops = src_iter->ops;
+       struct ttm_tt *ttm = bo->ttm;
+       struct dma_buf_map src_map, dst_map;
+       pgoff_t i;
 
-       if (mem->bus.addr) {
-               addr = mem->bus.addr;
-       } else {
-               size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT;
+       /* Single TTM move. NOP */
+       if (dst_ops->maps_tt && src_ops->maps_tt)
+               return;
 
-               if (mem->bus.caching == ttm_write_combined)
-                       addr = ioremap_wc(mem->bus.offset, bus_size);
-#ifdef CONFIG_X86
-               else if (mem->bus.caching == ttm_cached)
-                       addr = ioremap_cache(mem->bus.offset, bus_size);
-#endif
-               else
-                       addr = ioremap(mem->bus.offset, bus_size);
-               if (!addr) {
-                       ttm_mem_io_free(bdev, mem);
-                       return -ENOMEM;
+       /* Don't move nonexistent data. Clear destination instead. */
+       if (src_ops->maps_tt && (!ttm || !ttm_tt_is_populated(ttm))) {
+               if (ttm && !(ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC))
+                       return;
+
+               for (i = 0; i < num_pages; ++i) {
+                       dst_ops->map_local(dst_iter, &dst_map, i);
+                       if (dst_map.is_iomem)
+                               memset_io(dst_map.vaddr_iomem, 0, PAGE_SIZE);
+                       else
+                               memset(dst_map.vaddr, 0, PAGE_SIZE);
+                       if (dst_ops->unmap_local)
+                               dst_ops->unmap_local(dst_iter, &dst_map);
                }
+               return;
        }
-       *virtual = addr;
-       return 0;
-}
-
-static void ttm_resource_iounmap(struct ttm_device *bdev,
-                               struct ttm_resource *mem,
-                               void *virtual)
-{
-       if (virtual && mem->bus.addr == NULL)
-               iounmap(virtual);
-       ttm_mem_io_free(bdev, mem);
-}
-
-static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
-{
-       uint32_t *dstP =
-           (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
-       uint32_t *srcP =
-           (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
-
-       int i;
-       for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
-               iowrite32(ioread32(srcP++), dstP++);
-       return 0;
-}
-
-static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
-                               unsigned long page,
-                               pgprot_t prot)
-{
-       struct page *d = ttm->pages[page];
-       void *dst;
-
-       if (!d)
-               return -ENOMEM;
-
-       src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
-       dst = kmap_atomic_prot(d, prot);
-       if (!dst)
-               return -ENOMEM;
-
-       memcpy_fromio(dst, src, PAGE_SIZE);
-
-       kunmap_atomic(dst);
-
-       return 0;
-}
-
-static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
-                               unsigned long page,
-                               pgprot_t prot)
-{
-       struct page *s = ttm->pages[page];
-       void *src;
-
-       if (!s)
-               return -ENOMEM;
-
-       dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
-       src = kmap_atomic_prot(s, prot);
-       if (!src)
-               return -ENOMEM;
 
-       memcpy_toio(dst, src, PAGE_SIZE);
-
-       kunmap_atomic(src);
+       for (i = 0; i < num_pages; ++i) {
+               dst_ops->map_local(dst_iter, &dst_map, i);
+               src_ops->map_local(src_iter, &src_map, i);
+
+               if (!src_map.is_iomem && !dst_map.is_iomem) {
+                       memcpy(dst_map.vaddr, src_map.vaddr, PAGE_SIZE);
+               } else if (!src_map.is_iomem) {
+                       dma_buf_map_memcpy_to(&dst_map, src_map.vaddr,
+                                             PAGE_SIZE);
+               } else if (!dst_map.is_iomem) {
+                       memcpy_fromio(dst_map.vaddr, src_map.vaddr_iomem,
+                                     PAGE_SIZE);
+               } else {
+                       int j;
+                       u32 __iomem *src = src_map.vaddr_iomem;
+                       u32 __iomem *dst = dst_map.vaddr_iomem;
 
-       return 0;
+                       for (j = 0; j < (PAGE_SIZE / sizeof(u32)); ++j)
+                               iowrite32(ioread32(src++), dst++);
+               }
+               if (src_ops->unmap_local)
+                       src_ops->unmap_local(src_iter, &src_map);
+               if (dst_ops->unmap_local)
+                       dst_ops->unmap_local(dst_iter, &dst_map);
+       }
 }
+EXPORT_SYMBOL(ttm_move_memcpy);
 
 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
                       struct ttm_operation_ctx *ctx,
-                      struct ttm_resource *new_mem)
+                      struct ttm_resource *dst_mem)
 {
-       struct ttm_resource *old_mem = bo->resource;
        struct ttm_device *bdev = bo->bdev;
-       struct ttm_resource_manager *man;
+       struct ttm_resource_manager *dst_man =
+               ttm_manager_type(bo->bdev, dst_mem->mem_type);
        struct ttm_tt *ttm = bo->ttm;
-       void *old_iomap;
-       void *new_iomap;
-       int ret;
-       unsigned long i;
-
-       man = ttm_manager_type(bdev, new_mem->mem_type);
-
-       ret = ttm_bo_wait_ctx(bo, ctx);
-       if (ret)
-               return ret;
-
-       ret = ttm_resource_ioremap(bdev, old_mem, &old_iomap);
-       if (ret)
-               return ret;
-       ret = ttm_resource_ioremap(bdev, new_mem, &new_iomap);
-       if (ret)
-               goto out;
-
-       /*
-        * Single TTM move. NOP.
-        */
-       if (old_iomap == NULL && new_iomap == NULL)
-               goto out1;
-
-       /*
-        * Don't move nonexistent data. Clear destination instead.
-        */
-       if (old_iomap == NULL &&
-           (ttm == NULL || (!ttm_tt_is_populated(ttm) &&
-                            !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) {
-               memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE);
-               goto out1;
-       }
+       struct ttm_resource *src_mem = bo->resource;
+       struct ttm_resource_manager *src_man =
+               ttm_manager_type(bdev, src_mem->mem_type);
+       struct ttm_resource src_copy = *src_mem;
+       union {
+               struct ttm_kmap_iter_tt tt;
+               struct ttm_kmap_iter_linear_io io;
+       } _dst_iter, _src_iter;
+       struct ttm_kmap_iter *dst_iter, *src_iter;
+       int ret = 0;
 
-       /*
-        * TTM might be null for moves within the same region.
-        */
-       if (ttm) {
+       if (ttm && ((ttm->page_flags & TTM_PAGE_FLAG_SWAPPED) ||
+                   dst_man->use_tt)) {
                ret = ttm_tt_populate(bdev, ttm, ctx);
                if (ret)
-                       goto out1;
+                       return ret;
        }
 
-       for (i = 0; i < new_mem->num_pages; ++i) {
-               if (old_iomap == NULL) {
-                       pgprot_t prot = ttm_io_prot(bo, old_mem, PAGE_KERNEL);
-                       ret = ttm_copy_ttm_io_page(ttm, new_iomap, i,
-                                                  prot);
-               } else if (new_iomap == NULL) {
-                       pgprot_t prot = ttm_io_prot(bo, new_mem, PAGE_KERNEL);
-                       ret = ttm_copy_io_ttm_page(ttm, old_iomap, i,
-                                                  prot);
-               } else {
-                       ret = ttm_copy_io_page(new_iomap, old_iomap, i);
-               }
-               if (ret)
-                       break;
-       }
-       mb();
-out1:
-       ttm_resource_iounmap(bdev, new_mem, new_iomap);
-out:
-       ttm_resource_iounmap(bdev, old_mem, old_iomap);
-
-       if (ret) {
-               ttm_resource_free(bo, &new_mem);
-               return ret;
+       dst_iter = ttm_kmap_iter_linear_io_init(&_dst_iter.io, bdev, dst_mem);
+       if (PTR_ERR(dst_iter) == -EINVAL && dst_man->use_tt)
+               dst_iter = ttm_kmap_iter_tt_init(&_dst_iter.tt, bo->ttm);
+       if (IS_ERR(dst_iter))
+               return PTR_ERR(dst_iter);
+
+       src_iter = ttm_kmap_iter_linear_io_init(&_src_iter.io, bdev, src_mem);
+       if (PTR_ERR(src_iter) == -EINVAL && src_man->use_tt)
+               src_iter = ttm_kmap_iter_tt_init(&_src_iter.tt, bo->ttm);
+       if (IS_ERR(src_iter)) {
+               ret = PTR_ERR(src_iter);
+               goto out_src_iter;
        }
 
-       ttm_resource_free(bo, &bo->resource);
-       ttm_bo_assign_mem(bo, new_mem);
+       ttm_move_memcpy(bo, dst_mem->num_pages, dst_iter, src_iter);
+       src_copy = *src_mem;
+       ttm_bo_move_sync_cleanup(bo, dst_mem);
 
-       if (!man->use_tt)
-               ttm_bo_tt_destroy(bo);
+       if (!src_iter->ops->maps_tt)
+               ttm_kmap_iter_linear_io_fini(&_src_iter.io, bdev, &src_copy);
+out_src_iter:
+       if (!dst_iter->ops->maps_tt)
+               ttm_kmap_iter_linear_io_fini(&_dst_iter.io, bdev, dst_mem);
 
        return ret;
 }
@@ -335,27 +272,7 @@ pgprot_t ttm_io_prot(struct ttm_buffer_object *bo, struct ttm_resource *res,
        man = ttm_manager_type(bo->bdev, res->mem_type);
        caching = man->use_tt ? bo->ttm->caching : res->bus.caching;
 
-       /* Cached mappings need no adjustment */
-       if (caching == ttm_cached)
-               return tmp;
-
-#if defined(__i386__) || defined(__x86_64__)
-       if (caching == ttm_write_combined)
-               tmp = pgprot_writecombine(tmp);
-       else if (boot_cpu_data.x86 > 3)
-               tmp = pgprot_noncached(tmp);
-#endif
-#if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \
-    defined(__powerpc__) || defined(__mips__)
-       if (caching == ttm_write_combined)
-               tmp = pgprot_writecombine(tmp);
-       else
-               tmp = pgprot_noncached(tmp);
-#endif
-#if defined(__sparc__)
-       tmp = pgprot_noncached(tmp);
-#endif
-       return tmp;
+       return ttm_prot_from_caching(caching, tmp);
 }
 EXPORT_SYMBOL(ttm_io_prot);
 
index 56b0efdba1a97d8fd98a887c6ae85d3e0b6d7b68..997c458f68a9a6999d871f5cc33a2bdf0091ee31 100644 (file)
  */
 #include <linux/module.h>
 #include <linux/device.h>
+#include <linux/pgtable.h>
 #include <linux/sched.h>
 #include <linux/debugfs.h>
 #include <drm/drm_sysfs.h>
+#include <drm/ttm/ttm_caching.h>
 
 #include "ttm_module.h"
 
+/**
+ * ttm_prot_from_caching - Modify the page protection according to the
+ * ttm cacing mode
+ * @caching: The ttm caching mode
+ * @tmp: The original page protection
+ *
+ * Return: The modified page protection
+ */
+pgprot_t ttm_prot_from_caching(enum ttm_caching caching, pgprot_t tmp)
+{
+       /* Cached mappings need no adjustment */
+       if (caching == ttm_cached)
+               return tmp;
+
+#if defined(__i386__) || defined(__x86_64__)
+       if (caching == ttm_write_combined)
+               tmp = pgprot_writecombine(tmp);
+       else if (boot_cpu_data.x86 > 3)
+               tmp = pgprot_noncached(tmp);
+#endif
+#if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \
+       defined(__powerpc__) || defined(__mips__)
+       if (caching == ttm_write_combined)
+               tmp = pgprot_writecombine(tmp);
+       else
+               tmp = pgprot_noncached(tmp);
+#endif
+#if defined(__sparc__)
+       tmp = pgprot_noncached(tmp);
+#endif
+       return tmp;
+}
+
 struct dentry *ttm_debugfs_root;
 
 static int __init ttm_init(void)
index 2a68145572cc3646a7659b077641d23ac06bd7f1..2431717376e701e5693d3653e1af96d886e5f852 100644 (file)
  * Authors: Christian König
  */
 
+#include <linux/dma-buf-map.h>
+#include <linux/io-mapping.h>
+#include <linux/scatterlist.h>
+
 #include <drm/ttm/ttm_resource.h>
 #include <drm/ttm/ttm_bo_driver.h>
 
@@ -154,3 +158,192 @@ void ttm_resource_manager_debug(struct ttm_resource_manager *man,
                man->func->debug(man, p);
 }
 EXPORT_SYMBOL(ttm_resource_manager_debug);
+
+static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
+                                         struct dma_buf_map *dmap,
+                                         pgoff_t i)
+{
+       struct ttm_kmap_iter_iomap *iter_io =
+               container_of(iter, typeof(*iter_io), base);
+       void __iomem *addr;
+
+retry:
+       while (i >= iter_io->cache.end) {
+               iter_io->cache.sg = iter_io->cache.sg ?
+                       sg_next(iter_io->cache.sg) : iter_io->st->sgl;
+               iter_io->cache.i = iter_io->cache.end;
+               iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
+                       PAGE_SHIFT;
+               iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
+                       iter_io->start;
+       }
+
+       if (i < iter_io->cache.i) {
+               iter_io->cache.end = 0;
+               iter_io->cache.sg = NULL;
+               goto retry;
+       }
+
+       addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
+                                      (((resource_size_t)i - iter_io->cache.i)
+                                       << PAGE_SHIFT));
+       dma_buf_map_set_vaddr_iomem(dmap, addr);
+}
+
+static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
+                                           struct dma_buf_map *map)
+{
+       io_mapping_unmap_local(map->vaddr_iomem);
+}
+
+static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
+       .map_local =  ttm_kmap_iter_iomap_map_local,
+       .unmap_local = ttm_kmap_iter_iomap_unmap_local,
+       .maps_tt = false,
+};
+
+/**
+ * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
+ * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
+ * @iomap: The struct io_mapping representing the underlying linear io_memory.
+ * @st: sg_table into @iomap, representing the memory of the struct
+ * ttm_resource.
+ * @start: Offset that needs to be subtracted from @st to make
+ * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
+ *
+ * Return: Pointer to the embedded struct ttm_kmap_iter.
+ */
+struct ttm_kmap_iter *
+ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
+                        struct io_mapping *iomap,
+                        struct sg_table *st,
+                        resource_size_t start)
+{
+       iter_io->base.ops = &ttm_kmap_iter_io_ops;
+       iter_io->iomap = iomap;
+       iter_io->st = st;
+       iter_io->start = start;
+       memset(&iter_io->cache, 0, sizeof(iter_io->cache));
+
+       return &iter_io->base;
+}
+EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
+
+/**
+ * DOC: Linear io iterator
+ *
+ * This code should die in the not too near future. Best would be if we could
+ * make io-mapping use memremap for all io memory, and have memremap
+ * implement a kmap_local functionality. We could then strip a huge amount of
+ * code. These linear io iterators are implemented to mimic old functionality,
+ * and they don't use kmap_local semantics at all internally. Rather ioremap or
+ * friends, and at least on 32-bit they add global TLB flushes and points
+ * of failure.
+ */
+
+static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
+                                             struct dma_buf_map *dmap,
+                                             pgoff_t i)
+{
+       struct ttm_kmap_iter_linear_io *iter_io =
+               container_of(iter, typeof(*iter_io), base);
+
+       *dmap = iter_io->dmap;
+       dma_buf_map_incr(dmap, i * PAGE_SIZE);
+}
+
+static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
+       .map_local =  ttm_kmap_iter_linear_io_map_local,
+       .maps_tt = false,
+};
+
+/**
+ * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
+ * @iter_io: The iterator to initialize
+ * @bdev: The TTM device
+ * @mem: The ttm resource representing the iomap.
+ *
+ * This function is for internal TTM use only. It sets up a memcpy kmap iterator
+ * pointing at a linear chunk of io memory.
+ *
+ * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
+ * failure.
+ */
+struct ttm_kmap_iter *
+ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
+                            struct ttm_device *bdev,
+                            struct ttm_resource *mem)
+{
+       int ret;
+
+       ret = ttm_mem_io_reserve(bdev, mem);
+       if (ret)
+               goto out_err;
+       if (!mem->bus.is_iomem) {
+               ret = -EINVAL;
+               goto out_io_free;
+       }
+
+       if (mem->bus.addr) {
+               dma_buf_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
+               iter_io->needs_unmap = false;
+       } else {
+               size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT;
+
+               iter_io->needs_unmap = true;
+               memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
+               if (mem->bus.caching == ttm_write_combined)
+                       dma_buf_map_set_vaddr_iomem(&iter_io->dmap,
+                                                   ioremap_wc(mem->bus.offset,
+                                                              bus_size));
+               else if (mem->bus.caching == ttm_cached)
+                       dma_buf_map_set_vaddr(&iter_io->dmap,
+                                             memremap(mem->bus.offset, bus_size,
+                                                      MEMREMAP_WB |
+                                                      MEMREMAP_WT |
+                                                      MEMREMAP_WC));
+
+               /* If uncached requested or if mapping cached or wc failed */
+               if (dma_buf_map_is_null(&iter_io->dmap))
+                       dma_buf_map_set_vaddr_iomem(&iter_io->dmap,
+                                                   ioremap(mem->bus.offset,
+                                                           bus_size));
+
+               if (dma_buf_map_is_null(&iter_io->dmap)) {
+                       ret = -ENOMEM;
+                       goto out_io_free;
+               }
+       }
+
+       iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
+       return &iter_io->base;
+
+out_io_free:
+       ttm_mem_io_free(bdev, mem);
+out_err:
+       return ERR_PTR(ret);
+}
+
+/**
+ * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
+ * @iter_io: The iterator to initialize
+ * @bdev: The TTM device
+ * @mem: The ttm resource representing the iomap.
+ *
+ * This function is for internal TTM use only. It cleans up a memcpy kmap
+ * iterator initialized by ttm_kmap_iter_linear_io_init.
+ */
+void
+ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
+                            struct ttm_device *bdev,
+                            struct ttm_resource *mem)
+{
+       if (iter_io->needs_unmap && dma_buf_map_is_set(&iter_io->dmap)) {
+               if (iter_io->dmap.is_iomem)
+                       iounmap(iter_io->dmap.vaddr_iomem);
+               else
+                       memunmap(iter_io->dmap.vaddr);
+       }
+
+       ttm_mem_io_free(bdev, mem);
+}
index 682c840c9a51726b5300b8a69bc4ae1a8f32b177..24031a8acd2ddb0211744f1bf7456dbfb182f283 100644 (file)
@@ -433,3 +433,48 @@ void ttm_tt_mgr_init(unsigned long num_pages, unsigned long num_dma32_pages)
        if (!ttm_dma32_pages_limit)
                ttm_dma32_pages_limit = num_dma32_pages;
 }
+
+static void ttm_kmap_iter_tt_map_local(struct ttm_kmap_iter *iter,
+                                      struct dma_buf_map *dmap,
+                                      pgoff_t i)
+{
+       struct ttm_kmap_iter_tt *iter_tt =
+               container_of(iter, typeof(*iter_tt), base);
+
+       dma_buf_map_set_vaddr(dmap, kmap_local_page_prot(iter_tt->tt->pages[i],
+                                                        iter_tt->prot));
+}
+
+static void ttm_kmap_iter_tt_unmap_local(struct ttm_kmap_iter *iter,
+                                        struct dma_buf_map *map)
+{
+       kunmap_local(map->vaddr);
+}
+
+static const struct ttm_kmap_iter_ops ttm_kmap_iter_tt_ops = {
+       .map_local = ttm_kmap_iter_tt_map_local,
+       .unmap_local = ttm_kmap_iter_tt_unmap_local,
+       .maps_tt = true,
+};
+
+/**
+ * ttm_kmap_iter_tt_init - Initialize a struct ttm_kmap_iter_tt
+ * @iter_tt: The struct ttm_kmap_iter_tt to initialize.
+ * @tt: Struct ttm_tt holding page pointers of the struct ttm_resource.
+ *
+ * Return: Pointer to the embedded struct ttm_kmap_iter.
+ */
+struct ttm_kmap_iter *
+ttm_kmap_iter_tt_init(struct ttm_kmap_iter_tt *iter_tt,
+                     struct ttm_tt *tt)
+{
+       iter_tt->base.ops = &ttm_kmap_iter_tt_ops;
+       iter_tt->tt = tt;
+       if (tt)
+               iter_tt->prot = ttm_prot_from_caching(tt->caching, PAGE_KERNEL);
+       else
+               iter_tt->prot = PAGE_KERNEL;
+
+       return &iter_tt->base;
+}
+EXPORT_SYMBOL(ttm_kmap_iter_tt_init);
index b266971c1974280926ade69a80ff745b6f416e80..68d6069572aa82c65feeb312ad73a68a614e2557 100644 (file)
@@ -40,6 +40,7 @@
 #include <drm/ttm/ttm_device.h>
 
 #include "ttm_bo_api.h"
+#include "ttm_kmap_iter.h"
 #include "ttm_placement.h"
 #include "ttm_tt.h"
 #include "ttm_pool.h"
@@ -270,6 +271,23 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
                              bool pipeline,
                              struct ttm_resource *new_mem);
 
+/**
+ * ttm_bo_move_accel_cleanup.
+ *
+ * @bo: A pointer to a struct ttm_buffer_object.
+ * @new_mem: struct ttm_resource indicating where to move.
+ *
+ * Special case of ttm_bo_move_accel_cleanup where the bo is guaranteed
+ * by the caller to be idle. Typically used after memcpy buffer moves.
+ */
+static inline void ttm_bo_move_sync_cleanup(struct ttm_buffer_object *bo,
+                                           struct ttm_resource *new_mem)
+{
+       int ret = ttm_bo_move_accel_cleanup(bo, NULL, true, false, new_mem);
+
+       WARN_ON(ret);
+}
+
 /**
  * ttm_bo_pipeline_gutting.
  *
@@ -304,4 +322,14 @@ int ttm_bo_tt_bind(struct ttm_buffer_object *bo, struct ttm_resource *mem);
  */
 void ttm_bo_tt_destroy(struct ttm_buffer_object *bo);
 
+void ttm_move_memcpy(struct ttm_buffer_object *bo,
+                    u32 num_pages,
+                    struct ttm_kmap_iter *dst_iter,
+                    struct ttm_kmap_iter *src_iter);
+
+struct ttm_kmap_iter *
+ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
+                        struct io_mapping *iomap,
+                        struct sg_table *st,
+                        resource_size_t start);
 #endif
index a0b4a49fa4328bb0037beacbdd3d76b0dd00cc3b..3c9dd65f5aaff854ae4bfa36e51bb3d37cfe4a32 100644 (file)
@@ -33,4 +33,6 @@ enum ttm_caching {
        ttm_cached
 };
 
+pgprot_t ttm_prot_from_caching(enum ttm_caching caching, pgprot_t tmp);
+
 #endif
diff --git a/include/drm/ttm/ttm_kmap_iter.h b/include/drm/ttm/ttm_kmap_iter.h
new file mode 100644 (file)
index 0000000..8bb00fd
--- /dev/null
@@ -0,0 +1,61 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2021 Intel Corporation
+ */
+#ifndef __TTM_KMAP_ITER_H__
+#define __TTM_KMAP_ITER_H__
+
+#include <linux/types.h>
+
+struct ttm_kmap_iter;
+struct dma_buf_map;
+
+/**
+ * struct ttm_kmap_iter_ops - Ops structure for a struct
+ * ttm_kmap_iter.
+ * @maps_tt: Whether the iterator maps TT memory directly, as opposed
+ * mapping a TT through an aperture. Both these modes have
+ * struct ttm_resource_manager::use_tt set, but the latter typically
+ * returns is_iomem == true from ttm_mem_io_reserve.
+ */
+struct ttm_kmap_iter_ops {
+       /**
+        * kmap_local() - Map a PAGE_SIZE part of the resource using
+        * kmap_local semantics.
+        * @res_iter: Pointer to the struct ttm_kmap_iter representing
+        * the resource.
+        * @dmap: The struct dma_buf_map holding the virtual address after
+        * the operation.
+        * @i: The location within the resource to map. PAGE_SIZE granularity.
+        */
+       void (*map_local)(struct ttm_kmap_iter *res_iter,
+                         struct dma_buf_map *dmap, pgoff_t i);
+       /**
+        * unmap_local() - Unmap a PAGE_SIZE part of the resource previously
+        * mapped using kmap_local.
+        * @res_iter: Pointer to the struct ttm_kmap_iter representing
+        * the resource.
+        * @dmap: The struct dma_buf_map holding the virtual address after
+        * the operation.
+        */
+       void (*unmap_local)(struct ttm_kmap_iter *res_iter,
+                           struct dma_buf_map *dmap);
+       bool maps_tt;
+};
+
+/**
+ * struct ttm_kmap_iter - Iterator for kmap_local type operations on a
+ * resource.
+ * @ops: Pointer to the operations struct.
+ *
+ * This struct is intended to be embedded in a resource-specific specialization
+ * implementing operations for the resource.
+ *
+ * Nothing stops us from extending the operations to vmap, vmap_pfn etc,
+ * replacing some or parts of the ttm_bo_util. cpu-map functionality.
+ */
+struct ttm_kmap_iter {
+       const struct ttm_kmap_iter_ops *ops;
+};
+
+#endif /* __TTM_KMAP_ITER_H__ */
index 4abb95b9fd11c619a3ffb5fb2b49a7e163c86bf3..140b6b9a8bbe68ad8b436a063a208074dea12969 100644 (file)
 
 #include <linux/types.h>
 #include <linux/mutex.h>
+#include <linux/dma-buf-map.h>
 #include <linux/dma-fence.h>
 #include <drm/drm_print.h>
 #include <drm/ttm/ttm_caching.h>
+#include <drm/ttm/ttm_kmap_iter.h>
 
 #define TTM_MAX_BO_PRIORITY    4U
 
@@ -38,6 +40,10 @@ struct ttm_resource_manager;
 struct ttm_resource;
 struct ttm_place;
 struct ttm_buffer_object;
+struct dma_buf_map;
+struct io_mapping;
+struct sg_table;
+struct scatterlist;
 
 struct ttm_resource_manager_func {
        /**
@@ -167,6 +173,45 @@ struct ttm_resource {
        struct ttm_bus_placement bus;
 };
 
+/**
+ * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping +
+ * struct sg_table backed struct ttm_resource.
+ * @base: Embedded struct ttm_kmap_iter providing the usage interface.
+ * @iomap: struct io_mapping representing the underlying linear io_memory.
+ * @st: sg_table into @iomap, representing the memory of the struct ttm_resource.
+ * @start: Offset that needs to be subtracted from @st to make
+ * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
+ * @cache: Scatterlist traversal cache for fast lookups.
+ * @cache.sg: Pointer to the currently cached scatterlist segment.
+ * @cache.i: First index of @sg. PAGE_SIZE granularity.
+ * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity.
+ * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity.
+ */
+struct ttm_kmap_iter_iomap {
+       struct ttm_kmap_iter base;
+       struct io_mapping *iomap;
+       struct sg_table *st;
+       resource_size_t start;
+       struct {
+               struct scatterlist *sg;
+               pgoff_t i;
+               pgoff_t end;
+               pgoff_t offs;
+       } cache;
+};
+
+/**
+ * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io
+ * @base: The base iterator
+ * @dmap: Points to the starting address of the region
+ * @needs_unmap: Whether we need to unmap on fini
+ */
+struct ttm_kmap_iter_linear_io {
+       struct ttm_kmap_iter base;
+       struct dma_buf_map dmap;
+       bool needs_unmap;
+};
+
 /**
  * ttm_resource_manager_set_used
  *
@@ -231,4 +276,20 @@ int ttm_resource_manager_evict_all(struct ttm_device *bdev,
 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
                                struct drm_printer *p);
 
+struct ttm_kmap_iter *
+ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
+                        struct io_mapping *iomap,
+                        struct sg_table *st,
+                        resource_size_t start);
+
+struct ttm_kmap_iter_linear_io;
+
+struct ttm_kmap_iter *
+ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
+                            struct ttm_device *bdev,
+                            struct ttm_resource *mem);
+
+void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
+                                 struct ttm_device *bdev,
+                                 struct ttm_resource *mem);
 #endif
index 134d09ef7766f1c38403f33747efa8e5868b12bf..3102059db726ce3fedb6dd1ba7cf32b3e60ad0a8 100644 (file)
@@ -29,6 +29,7 @@
 
 #include <linux/types.h>
 #include <drm/ttm/ttm_caching.h>
+#include <drm/ttm/ttm_kmap_iter.h>
 
 struct ttm_bo_device;
 struct ttm_tt;
@@ -69,6 +70,18 @@ struct ttm_tt {
        enum ttm_caching caching;
 };
 
+/**
+ * struct ttm_kmap_iter_tt - Specialization of a mappig iterator for a tt.
+ * @base: Embedded struct ttm_kmap_iter providing the usage interface
+ * @tt: Cached struct ttm_tt.
+ * @prot: Cached page protection for mapping.
+ */
+struct ttm_kmap_iter_tt {
+       struct ttm_kmap_iter base;
+       struct ttm_tt *tt;
+       pgprot_t prot;
+};
+
 static inline bool ttm_tt_is_populated(struct ttm_tt *tt)
 {
        return tt->page_flags & TTM_PAGE_FLAG_PRIV_POPULATED;
@@ -159,6 +172,9 @@ void ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm);
 
 void ttm_tt_mgr_init(unsigned long num_pages, unsigned long num_dma32_pages);
 
+struct ttm_kmap_iter *ttm_kmap_iter_tt_init(struct ttm_kmap_iter_tt *iter_tt,
+                                           struct ttm_tt *tt);
+
 #if IS_ENABLED(CONFIG_AGP)
 #include <linux/agp_backend.h>