block: allow using the per-cpu bio cache from bio_alloc_bioset

Replace the BIO_PERCPU_CACHE bio-internal flag with a REQ_ALLOC_CACHE
one that can be passed to bio_alloc / bio_alloc_bioset, and implement
the percpu cache allocation logic in a helper called from
bio_alloc_bioset.  This allows any bio_alloc_bioset user to use the
percpu caches instead of having the functionality tied to struct kiocb.

Signed-off-by: Mike Snitzer <snitzer@kernel.org>
[hch: refactored a bit]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Link: https://lore.kernel.org/r/20220324203526.62306-2-snitzer@kernel.org
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/bio.c b/block/bio.c
index 4259125e16ab24075d902f989f424f8b046c9bee..c67f47c6cc8ca56f31e77d6474d9ab76de2d92fb 100644 (file)
--- a/block/bio.c
+++ b/block/bio.c
@@ -419,6 +419,28 @@ static void punt_bios_to_rescuer(struct bio_set *bs)
        queue_work(bs->rescue_workqueue, &bs->rescue_work);
 }
 
+static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
+               unsigned short nr_vecs, unsigned int opf, gfp_t gfp,
+               struct bio_set *bs)
+{
+       struct bio_alloc_cache *cache;
+       struct bio *bio;
+
+       cache = per_cpu_ptr(bs->cache, get_cpu());
+       if (!cache->free_list) {
+               put_cpu();
+               return NULL;
+       }
+       bio = cache->free_list;
+       cache->free_list = bio->bi_next;
+       cache->nr--;
+       put_cpu();
+
+       bio_init(bio, bdev, nr_vecs ? bio->bi_inline_vecs : NULL, nr_vecs, opf);
+       bio->bi_pool = bs;
+       return bio;
+}
+
 /**
  * bio_alloc_bioset - allocate a bio for I/O
  * @bdev:      block device to allocate the bio for (can be %NULL)
@@ -451,6 +473,9 @@ static void punt_bios_to_rescuer(struct bio_set *bs)
  * submit_bio_noacct() should be avoided - instead, use bio_set's front_pad
  * for per bio allocations.
  *
+ * If REQ_ALLOC_CACHE is set, the final put of the bio MUST be done from process
+ * context, not hard/soft IRQ.
+ *
  * Returns: Pointer to new bio on success, NULL on failure.
  */
 struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
@@ -465,6 +490,21 @@ struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
        if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_vecs > 0))
                return NULL;
 
+       if (opf & REQ_ALLOC_CACHE) {
+               if (bs->cache && nr_vecs <= BIO_INLINE_VECS) {
+                       bio = bio_alloc_percpu_cache(bdev, nr_vecs, opf,
+                                                    gfp_mask, bs);
+                       if (bio)
+                               return bio;
+                       /*
+                        * No cached bio available, bio returned below marked with
+                        * REQ_ALLOC_CACHE to particpate in per-cpu alloc cache.
+                        */
+               } else {
+                       opf &= ~REQ_ALLOC_CACHE;
+               }
+       }
+
        /*
         * submit_bio_noacct() converts recursion to iteration; this means if
         * we're running beneath it, any bios we allocate and submit will not be
@@ -711,7 +751,7 @@ void bio_put(struct bio *bio)
                        return;
        }
 
-       if (bio_flagged(bio, BIO_PERCPU_CACHE)) {
+       if (bio->bi_opf & REQ_ALLOC_CACHE) {
                struct bio_alloc_cache *cache;
 
                bio_uninit(bio);
@@ -1732,50 +1772,6 @@ int bioset_init_from_src(struct bio_set *bs, struct bio_set *src)
 }
 EXPORT_SYMBOL(bioset_init_from_src);
 
-/**
- * bio_alloc_kiocb - Allocate a bio from bio_set based on kiocb
- * @kiocb:     kiocb describing the IO
- * @bdev:      block device to allocate the bio for (can be %NULL)
- * @nr_vecs:   number of iovecs to pre-allocate
- * @opf:       operation and flags for bio
- * @bs:                bio_set to allocate from
- *
- * Description:
- *    Like @bio_alloc_bioset, but pass in the kiocb. The kiocb is only
- *    used to check if we should dip into the per-cpu bio_set allocation
- *    cache. The allocation uses GFP_KERNEL internally. On return, the
- *    bio is marked BIO_PERCPU_CACHEABLE, and the final put of the bio
- *    MUST be done from process context, not hard/soft IRQ.
- *
- */
-struct bio *bio_alloc_kiocb(struct kiocb *kiocb, struct block_device *bdev,
-               unsigned short nr_vecs, unsigned int opf, struct bio_set *bs)
-{
-       struct bio_alloc_cache *cache;
-       struct bio *bio;
-
-       if (!(kiocb->ki_flags & IOCB_ALLOC_CACHE) || nr_vecs > BIO_INLINE_VECS)
-               return bio_alloc_bioset(bdev, nr_vecs, opf, GFP_KERNEL, bs);
-
-       cache = per_cpu_ptr(bs->cache, get_cpu());
-       if (cache->free_list) {
-               bio = cache->free_list;
-               cache->free_list = bio->bi_next;
-               cache->nr--;
-               put_cpu();
-               bio_init(bio, bdev, nr_vecs ? bio->bi_inline_vecs : NULL,
-                        nr_vecs, opf);
-               bio->bi_pool = bs;
-               bio_set_flag(bio, BIO_PERCPU_CACHE);
-               return bio;
-       }
-       put_cpu();
-       bio = bio_alloc_bioset(bdev, nr_vecs, opf, GFP_KERNEL, bs);
-       bio_set_flag(bio, BIO_PERCPU_CACHE);
-       return bio;
-}
-EXPORT_SYMBOL_GPL(bio_alloc_kiocb);
-
 static int __init init_bio(void)
 {
        int i;

diff --git a/block/blk.h b/block/blk.h
index 8ccbc6e076369b753f21f56b62f8baf4e4e4ef61..0d33e4ca2c520de8678a1ef2882d6f5531ff43d6 100644 (file)
--- a/block/blk.h
+++ b/block/blk.h
@@ -453,8 +453,7 @@ extern struct device_attribute dev_attr_events_poll_msecs;
 static inline void bio_clear_polled(struct bio *bio)
 {
        /* can't support alloc cache if we turn off polling */
-       bio_clear_flag(bio, BIO_PERCPU_CACHE);
-       bio->bi_opf &= ~REQ_POLLED;
+       bio->bi_opf &= ~(REQ_POLLED | REQ_ALLOC_CACHE);
 }
 
 long blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg);

diff --git a/block/fops.c b/block/fops.c
index 9f2ecec406b04e51f305e997c090176db70ce90b..ba5e7d5ff9a52e03e4bae3f838da82f55e41581c 100644 (file)
--- a/block/fops.c
+++ b/block/fops.c
@@ -197,8 +197,10 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
            (bdev_logical_block_size(bdev) - 1))
                return -EINVAL;
 
-       bio = bio_alloc_kiocb(iocb, bdev, nr_pages, opf, &blkdev_dio_pool);
-
+       if (iocb->ki_flags & IOCB_ALLOC_CACHE)
+               opf |= REQ_ALLOC_CACHE;
+       bio = bio_alloc_bioset(bdev, nr_pages, opf, GFP_KERNEL,
+                              &blkdev_dio_pool);
        dio = container_of(bio, struct blkdev_dio, bio);
        atomic_set(&dio->ref, 1);
        /*
@@ -320,7 +322,10 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb,
            (bdev_logical_block_size(bdev) - 1))
                return -EINVAL;
 
-       bio = bio_alloc_kiocb(iocb, bdev, nr_pages, opf, &blkdev_dio_pool);
+       if (iocb->ki_flags & IOCB_ALLOC_CACHE)
+               opf |= REQ_ALLOC_CACHE;
+       bio = bio_alloc_bioset(bdev, nr_pages, opf, GFP_KERNEL,
+                              &blkdev_dio_pool);
        dio = container_of(bio, struct blkdev_dio, bio);
        dio->flags = 0;
        dio->iocb = iocb;

diff --git a/include/linux/bio.h b/include/linux/bio.h
index 278cc81cc1e7fa7c264bcdda2c7e1dacbeedf95b..0fe8a51a0778e60074cdd729fccdcbce64cc3f58 100644 (file)
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -405,8 +405,6 @@ extern int bioset_init_from_src(struct bio_set *bs, struct bio_set *src);
 struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
                             unsigned int opf, gfp_t gfp_mask,
                             struct bio_set *bs);
-struct bio *bio_alloc_kiocb(struct kiocb *kiocb, struct block_device *bdev,
-               unsigned short nr_vecs, unsigned int opf, struct bio_set *bs);
 struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned short nr_iovecs);
 extern void bio_put(struct bio *);
 

diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index 1973ef9bd40fcfbdff0bc7817272e8963f59c80a..4968cb17b13b48a829e8c7cc44c7f4d28d562ac8 100644 (file)
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -329,7 +329,6 @@ enum {
        BIO_QOS_MERGED,         /* but went through rq_qos merge path */
        BIO_REMAPPED,
        BIO_ZONE_WRITE_LOCKED,  /* Owns a zoned device zone write lock */
-       BIO_PERCPU_CACHE,       /* can participate in per-cpu alloc cache */
        BIO_FLAG_LAST
 };
 
@@ -414,6 +413,7 @@ enum req_flag_bits {
        __REQ_NOUNMAP,          /* do not free blocks when zeroing */
 
        __REQ_POLLED,           /* caller polls for completion using bio_poll */
+       __REQ_ALLOC_CACHE,      /* allocate IO from cache if available */
 
        /* for driver use */
        __REQ_DRV,
@@ -439,6 +439,7 @@ enum req_flag_bits {
 
 #define REQ_NOUNMAP            (1ULL << __REQ_NOUNMAP)
 #define REQ_POLLED             (1ULL << __REQ_POLLED)
+#define REQ_ALLOC_CACHE                (1ULL << __REQ_ALLOC_CACHE)
 
 #define REQ_DRV                        (1ULL << __REQ_DRV)
 #define REQ_SWAP               (1ULL << __REQ_SWAP)