xsk: Enable sharing of dma mappings

Enable the sharing of dma mappings by moving them out from the buffer
pool. Instead we put each dma mapped umem region in a list in the umem
structure. If dma has already been mapped for this umem and device, it
is not mapped again and the existing dma mappings are reused.

Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Björn Töpel <bjorn.topel@intel.com>
Link: https://lore.kernel.org/bpf/1598603189-32145-9-git-send-email-magnus.karlsson@intel.com

diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
index 126d24364b5a28e73878429add0be484a078221f..282aeba0d20f89b0098820b0d5f93a2fa53ed3d9 100644 (file)
--- a/include/net/xdp_sock.h
+++ b/include/net/xdp_sock.h
@@ -30,6 +30,7 @@ struct xdp_umem {
        u8 flags;
        int id;
        bool zc;
+       struct list_head xsk_dma_list;
 };
 
 struct xsk_map {

diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h
index 83f100c6d4402455246c822702ab4405cbfadb62..356d0ac74eba5c10a3a209a43895c0734b78625c 100644 (file)
--- a/include/net/xsk_buff_pool.h
+++ b/include/net/xsk_buff_pool.h
@@ -28,10 +28,23 @@ struct xdp_buff_xsk {
        struct list_head free_list_node;
 };
 
+struct xsk_dma_map {
+       dma_addr_t *dma_pages;
+       struct device *dev;
+       struct net_device *netdev;
+       refcount_t users;
+       struct list_head list; /* Protected by the RTNL_LOCK */
+       u32 dma_pages_cnt;
+       bool dma_need_sync;
+};
+
 struct xsk_buff_pool {
        struct xsk_queue *fq;
        struct xsk_queue *cq;
        struct list_head free_list;
+       /* For performance reasons, each buff pool has its own array of dma_pages
+        * even when they are identical.
+        */
        dma_addr_t *dma_pages;
        struct xdp_buff_xsk *heads;
        u64 chunk_mask;

diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c
index 77604c30aa0f86da1149cfa8d77c0099d2c691da..a7227b447228dadbf9f76dd18de4cd50f784aa17 100644 (file)
--- a/net/xdp/xdp_umem.c
+++ b/net/xdp/xdp_umem.c
@@ -198,6 +198,7 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
        umem->user = NULL;
        umem->flags = mr->flags;
 
+       INIT_LIST_HEAD(&umem->xsk_dma_list);
        refcount_set(&umem->users, 1);
 
        err = xdp_umem_account_pages(umem);

diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c
index c56387439f679c2beb634fa5303b711b3e54eea7..547eb41f78371ac585372e546099c6428ee7a647 100644 (file)
--- a/net/xdp/xsk_buff_pool.c
+++ b/net/xdp/xsk_buff_pool.c
@@ -104,6 +104,25 @@ void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq)
 }
 EXPORT_SYMBOL(xp_set_rxq_info);
 
+static void xp_disable_drv_zc(struct xsk_buff_pool *pool)
+{
+       struct netdev_bpf bpf;
+       int err;
+
+       ASSERT_RTNL();
+
+       if (pool->umem->zc) {
+               bpf.command = XDP_SETUP_XSK_POOL;
+               bpf.xsk.pool = NULL;
+               bpf.xsk.queue_id = pool->queue_id;
+
+               err = pool->netdev->netdev_ops->ndo_bpf(pool->netdev, &bpf);
+
+               if (err)
+                       WARN(1, "Failed to disable zero-copy!\n");
+       }
+}
+
 int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *netdev,
                  u16 queue_id, u16 flags)
 {
@@ -122,6 +141,8 @@ int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *netdev,
        if (xsk_get_pool_from_qid(netdev, queue_id))
                return -EBUSY;
 
+       pool->netdev = netdev;
+       pool->queue_id = queue_id;
        err = xsk_reg_pool_at_qid(netdev, pool, queue_id);
        if (err)
                return err;
@@ -155,11 +176,15 @@ int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *netdev,
        if (err)
                goto err_unreg_pool;
 
-       pool->netdev = netdev;
-       pool->queue_id = queue_id;
+       if (!pool->dma_pages) {
+               WARN(1, "Driver did not DMA map zero-copy buffers");
+               goto err_unreg_xsk;
+       }
        pool->umem->zc = true;
        return 0;
 
+err_unreg_xsk:
+       xp_disable_drv_zc(pool);
 err_unreg_pool:
        if (!force_zc)
                err = 0; /* fallback to copy mode */
@@ -170,25 +195,10 @@ err_unreg_pool:
 
 void xp_clear_dev(struct xsk_buff_pool *pool)
 {
-       struct netdev_bpf bpf;
-       int err;
-
-       ASSERT_RTNL();
-
        if (!pool->netdev)
                return;
 
-       if (pool->umem->zc) {
-               bpf.command = XDP_SETUP_XSK_POOL;
-               bpf.xsk.pool = NULL;
-               bpf.xsk.queue_id = pool->queue_id;
-
-               err = pool->netdev->netdev_ops->ndo_bpf(pool->netdev, &bpf);
-
-               if (err)
-                       WARN(1, "Failed to disable zero-copy!\n");
-       }
-
+       xp_disable_drv_zc(pool);
        xsk_clear_pool_at_qid(pool->netdev, pool->queue_id);
        dev_put(pool->netdev);
        pool->netdev = NULL;
@@ -233,70 +243,159 @@ void xp_put_pool(struct xsk_buff_pool *pool)
        }
 }
 
-void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs)
+static struct xsk_dma_map *xp_find_dma_map(struct xsk_buff_pool *pool)
+{
+       struct xsk_dma_map *dma_map;
+
+       list_for_each_entry(dma_map, &pool->umem->xsk_dma_list, list) {
+               if (dma_map->netdev == pool->netdev)
+                       return dma_map;
+       }
+
+       return NULL;
+}
+
+static struct xsk_dma_map *xp_create_dma_map(struct device *dev, struct net_device *netdev,
+                                            u32 nr_pages, struct xdp_umem *umem)
+{
+       struct xsk_dma_map *dma_map;
+
+       dma_map = kzalloc(sizeof(*dma_map), GFP_KERNEL);
+       if (!dma_map)
+               return NULL;
+
+       dma_map->dma_pages = kvcalloc(nr_pages, sizeof(*dma_map->dma_pages), GFP_KERNEL);
+       if (!dma_map) {
+               kfree(dma_map);
+               return NULL;
+       }
+
+       dma_map->netdev = netdev;
+       dma_map->dev = dev;
+       dma_map->dma_need_sync = false;
+       dma_map->dma_pages_cnt = nr_pages;
+       refcount_set(&dma_map->users, 0);
+       list_add(&dma_map->list, &umem->xsk_dma_list);
+       return dma_map;
+}
+
+static void xp_destroy_dma_map(struct xsk_dma_map *dma_map)
+{
+       list_del(&dma_map->list);
+       kvfree(dma_map->dma_pages);
+       kfree(dma_map);
+}
+
+static void __xp_dma_unmap(struct xsk_dma_map *dma_map, unsigned long attrs)
 {
        dma_addr_t *dma;
        u32 i;
 
-       if (pool->dma_pages_cnt == 0)
-               return;
-
-       for (i = 0; i < pool->dma_pages_cnt; i++) {
-               dma = &pool->dma_pages[i];
+       for (i = 0; i < dma_map->dma_pages_cnt; i++) {
+               dma = &dma_map->dma_pages[i];
                if (*dma) {
-                       dma_unmap_page_attrs(pool->dev, *dma, PAGE_SIZE,
+                       dma_unmap_page_attrs(dma_map->dev, *dma, PAGE_SIZE,
                                             DMA_BIDIRECTIONAL, attrs);
                        *dma = 0;
                }
        }
 
+       xp_destroy_dma_map(dma_map);
+}
+
+void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs)
+{
+       struct xsk_dma_map *dma_map;
+
+       if (pool->dma_pages_cnt == 0)
+               return;
+
+       dma_map = xp_find_dma_map(pool);
+       if (!dma_map) {
+               WARN(1, "Could not find dma_map for device");
+               return;
+       }
+
+       if (!refcount_dec_and_test(&dma_map->users))
+               return;
+
+       __xp_dma_unmap(dma_map, attrs);
        kvfree(pool->dma_pages);
        pool->dma_pages_cnt = 0;
        pool->dev = NULL;
 }
 EXPORT_SYMBOL(xp_dma_unmap);
 
-static void xp_check_dma_contiguity(struct xsk_buff_pool *pool)
+static void xp_check_dma_contiguity(struct xsk_dma_map *dma_map)
 {
        u32 i;
 
-       for (i = 0; i < pool->dma_pages_cnt - 1; i++) {
-               if (pool->dma_pages[i] + PAGE_SIZE == pool->dma_pages[i + 1])
-                       pool->dma_pages[i] |= XSK_NEXT_PG_CONTIG_MASK;
+       for (i = 0; i < dma_map->dma_pages_cnt - 1; i++) {
+               if (dma_map->dma_pages[i] + PAGE_SIZE == dma_map->dma_pages[i + 1])
+                       dma_map->dma_pages[i] |= XSK_NEXT_PG_CONTIG_MASK;
                else
-                       pool->dma_pages[i] &= ~XSK_NEXT_PG_CONTIG_MASK;
+                       dma_map->dma_pages[i] &= ~XSK_NEXT_PG_CONTIG_MASK;
        }
 }
 
+static int xp_init_dma_info(struct xsk_buff_pool *pool, struct xsk_dma_map *dma_map)
+{
+       pool->dma_pages = kvcalloc(dma_map->dma_pages_cnt, sizeof(*pool->dma_pages), GFP_KERNEL);
+       if (!pool->dma_pages)
+               return -ENOMEM;
+
+       pool->dev = dma_map->dev;
+       pool->dma_pages_cnt = dma_map->dma_pages_cnt;
+       pool->dma_need_sync = dma_map->dma_need_sync;
+       refcount_inc(&dma_map->users);
+       memcpy(pool->dma_pages, dma_map->dma_pages,
+              pool->dma_pages_cnt * sizeof(*pool->dma_pages));
+
+       return 0;
+}
+
 int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev,
               unsigned long attrs, struct page **pages, u32 nr_pages)
 {
+       struct xsk_dma_map *dma_map;
        dma_addr_t dma;
+       int err;
        u32 i;
 
-       pool->dma_pages = kvcalloc(nr_pages, sizeof(*pool->dma_pages),
-                                  GFP_KERNEL);
-       if (!pool->dma_pages)
-               return -ENOMEM;
+       dma_map = xp_find_dma_map(pool);
+       if (dma_map) {
+               err = xp_init_dma_info(pool, dma_map);
+               if (err)
+                       return err;
 
-       pool->dev = dev;
-       pool->dma_pages_cnt = nr_pages;
-       pool->dma_need_sync = false;
+               return 0;
+       }
 
-       for (i = 0; i < pool->dma_pages_cnt; i++) {
+       dma_map = xp_create_dma_map(dev, pool->netdev, nr_pages, pool->umem);
+       if (!dma_map)
+               return -ENOMEM;
+
+       for (i = 0; i < dma_map->dma_pages_cnt; i++) {
                dma = dma_map_page_attrs(dev, pages[i], 0, PAGE_SIZE,
                                         DMA_BIDIRECTIONAL, attrs);
                if (dma_mapping_error(dev, dma)) {
-                       xp_dma_unmap(pool, attrs);
+                       __xp_dma_unmap(dma_map, attrs);
                        return -ENOMEM;
                }
                if (dma_need_sync(dev, dma))
-                       pool->dma_need_sync = true;
-               pool->dma_pages[i] = dma;
+                       dma_map->dma_need_sync = true;
+               dma_map->dma_pages[i] = dma;
        }
 
        if (pool->unaligned)
-               xp_check_dma_contiguity(pool);
+               xp_check_dma_contiguity(dma_map);
+
+       err = xp_init_dma_info(pool, dma_map);
+       if (err) {
+               __xp_dma_unmap(dma_map, attrs);
+               return err;
+       }
+
        return 0;
 }
 EXPORT_SYMBOL(xp_dma_map);