io_uring: add io_pin_pages() helper

Abstract this out from io_sqe_buffer_register() so we can use it
elsewhere too without duplicating this code.

No intended functional changes in this patch.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index dc129c74ed1e5931af1b6d704c8476dc3a121d91..384cdbd4094114894c63e23bbbc9dd54d08766b8 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -9845,30 +9845,18 @@ static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
        return ret;
 }
 
-static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
-                                 struct io_mapped_ubuf **pimu,
-                                 struct page **last_hpage)
+static struct page **io_pin_pages(unsigned long ubuf, unsigned long len,
+                                 int *npages)
 {
-       struct io_mapped_ubuf *imu = NULL;
+       unsigned long start, end, nr_pages;
        struct vm_area_struct **vmas = NULL;
        struct page **pages = NULL;
-       unsigned long off, start, end, ubuf;
-       size_t size;
-       int ret, pret, nr_pages, i;
-
-       if (!iov->iov_base) {
-               *pimu = ctx->dummy_ubuf;
-               return 0;
-       }
+       int i, pret, ret = -ENOMEM;
 
-       ubuf = (unsigned long) iov->iov_base;
-       end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+       end = (ubuf + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
        start = ubuf >> PAGE_SHIFT;
        nr_pages = end - start;
 
-       *pimu = NULL;
-       ret = -ENOMEM;
-
        pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
        if (!pages)
                goto done;
@@ -9878,10 +9866,6 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
        if (!vmas)
                goto done;
 
-       imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
-       if (!imu)
-               goto done;
-
        ret = 0;
        mmap_read_lock(current->mm);
        pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
@@ -9899,6 +9883,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
                                break;
                        }
                }
+               *npages = nr_pages;
        } else {
                ret = pret < 0 ? pret : -EFAULT;
        }
@@ -9912,14 +9897,53 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
                        unpin_user_pages(pages, pret);
                goto done;
        }
+       ret = 0;
+done:
+       kvfree(vmas);
+       if (ret < 0) {
+               kvfree(pages);
+               pages = ERR_PTR(ret);
+       }
+       return pages;
+}
 
-       ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage);
+static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
+                                 struct io_mapped_ubuf **pimu,
+                                 struct page **last_hpage)
+{
+       struct io_mapped_ubuf *imu = NULL;
+       struct page **pages = NULL;
+       unsigned long off;
+       size_t size;
+       int ret, nr_pages, i;
+
+       if (!iov->iov_base) {
+               *pimu = ctx->dummy_ubuf;
+               return 0;
+       }
+
+       *pimu = NULL;
+       ret = -ENOMEM;
+
+       pages = io_pin_pages((unsigned long) iov->iov_base, iov->iov_len,
+                               &nr_pages);
+       if (IS_ERR(pages)) {
+               ret = PTR_ERR(pages);
+               pages = NULL;
+               goto done;
+       }
+
+       imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
+       if (!imu)
+               goto done;
+
+       ret = io_buffer_account_pin(ctx, pages, nr_pages, imu, last_hpage);
        if (ret) {
-               unpin_user_pages(pages, pret);
+               unpin_user_pages(pages, nr_pages);
                goto done;
        }
 
-       off = ubuf & ~PAGE_MASK;
+       off = (unsigned long) iov->iov_base & ~PAGE_MASK;
        size = iov->iov_len;
        for (i = 0; i < nr_pages; i++) {
                size_t vec_len;
@@ -9932,8 +9956,8 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
                size -= vec_len;
        }
        /* store original address for later verification */
-       imu->ubuf = ubuf;
-       imu->ubuf_end = ubuf + iov->iov_len;
+       imu->ubuf = (unsigned long) iov->iov_base;
+       imu->ubuf_end = imu->ubuf + iov->iov_len;
        imu->nr_bvecs = nr_pages;
        *pimu = imu;
        ret = 0;
@@ -9941,7 +9965,6 @@ done:
        if (ret)
                kvfree(imu);
        kvfree(pages);
-       kvfree(vmas);
        return ret;
 }