block: Delay default elevator initialization

When elevator_init_mq() is called from blk_mq_init_allocated_queue(),
the only information known about the device is the number of hardware
queues as the block device scan by the device driver is not completed
yet for most drivers. The device type and elevator required features
are not set yet, preventing to correctly select the default elevator
most suitable for the device.

This currently affects all multi-queue zoned block devices which default
to the "none" elevator instead of the required "mq-deadline" elevator.
These drives currently include host-managed SMR disks connected to a
smartpqi HBA and null_blk block devices with zoned mode enabled.
Upcoming NVMe Zoned Namespace devices will also be affected.

Fix this by adding the boolean elevator_init argument to
blk_mq_init_allocated_queue() to control the execution of
elevator_init_mq(). Two cases exist:
1) elevator_init = false is used for calls to
   blk_mq_init_allocated_queue() within blk_mq_init_queue(). In this
   case, a call to elevator_init_mq() is added to __device_add_disk(),
   resulting in the delayed initialization of the queue elevator
   after the device driver finished probing the device information. This
   effectively allows elevator_init_mq() access to more information
   about the device.
2) elevator_init = true preserves the current behavior of initializing
   the elevator directly from blk_mq_init_allocated_queue(). This case
   is used for the special request based DM devices where the device
   gendisk is created before the queue initialization and device
   information (e.g. queue limits) is already known when the queue
   initialization is executed.

Additionally, to make sure that the elevator initialization is never
done while requests are in-flight (there should be none when the device
driver calls device_add_disk()), freeze and quiesce the device request
queue before calling blk_mq_init_sched() in elevator_init_mq().

Reviewed-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Damien Le Moal <damien.lemoal@wdc.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-mq.c b/block/blk-mq.c
index d10a7ab4207a15bf9f3efa4b09e93ee8b54e2d60..3647776a0f6eefeaef8d9d0afe6ea9ef3eebbe11 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -2695,7 +2695,11 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
        if (!uninit_q)
                return ERR_PTR(-ENOMEM);
 
-       q = blk_mq_init_allocated_queue(set, uninit_q);
+       /*
+        * Initialize the queue without an elevator. device_add_disk() will do
+        * the initialization.
+        */
+       q = blk_mq_init_allocated_queue(set, uninit_q, false);
        if (IS_ERR(q))
                blk_cleanup_queue(uninit_q);
 
@@ -2846,7 +2850,8 @@ static unsigned int nr_hw_queues(struct blk_mq_tag_set *set)
 }
 
 struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
-                                                 struct request_queue *q)
+                                                 struct request_queue *q,
+                                                 bool elevator_init)
 {
        /* mark the queue as mq asap */
        q->mq_ops = set->ops;
@@ -2908,7 +2913,8 @@ struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
        blk_mq_add_queue_tag_set(set, q);
        blk_mq_map_swqueue(q);
 
-       elevator_init_mq(q);
+       if (elevator_init)
+               elevator_init_mq(q);
 
        return q;
 

diff --git a/block/elevator.c b/block/elevator.c
index 520d6b224b74cd3c976f8ac30afb481abdda817b..096a670d22d7009a050a30316cc83c72b33b2829 100644 (file)
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -712,7 +712,14 @@ void elevator_init_mq(struct request_queue *q)
        if (!e)
                return;
 
+       blk_mq_freeze_queue(q);
+       blk_mq_quiesce_queue(q);
+
        err = blk_mq_init_sched(q, e);
+
+       blk_mq_unquiesce_queue(q);
+       blk_mq_unfreeze_queue(q);
+
        if (err) {
                pr_warn("\"%s\" elevator initialization failed, "
                        "falling back to \"none\"\n", e->elevator_name);

diff --git a/block/genhd.c b/block/genhd.c
index 54f1f0d381f4d28c7b4d07d4044e4f0f7d2ddb05..26b31fcae217fd3d936ccb32c6d7847ff5654805 100644 (file)
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -695,6 +695,15 @@ static void __device_add_disk(struct device *parent, struct gendisk *disk,
        dev_t devt;
        int retval;
 
+       /*
+        * The disk queue should now be all set with enough information about
+        * the device for the elevator code to pick an adequate default
+        * elevator if one is needed, that is, for devices requesting queue
+        * registration.
+        */
+       if (register_queue)
+               elevator_init_mq(disk->queue);
+
        /* minors == 0 indicates to use ext devt from part0 and should
         * be accompanied with EXT_DEVT flag.  Make sure all
         * parameters make sense.

diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c
index 21d5c1784d0ce9c0145479be393a96ea08f6e9fb..3f8577e2c13be0440d7af403cb8498d8a4b78afa 100644 (file)
--- a/drivers/md/dm-rq.c
+++ b/drivers/md/dm-rq.c
@@ -563,7 +563,7 @@ int dm_mq_init_request_queue(struct mapped_device *md, struct dm_table *t)
        if (err)
                goto out_kfree_tag_set;
 
-       q = blk_mq_init_allocated_queue(md->tag_set, md->queue);
+       q = blk_mq_init_allocated_queue(md->tag_set, md->queue, true);
        if (IS_ERR(q)) {
                err = PTR_ERR(q);
                goto out_tag_set;

diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 62a3bb715899934c87454349b0a7a40dab040459..0bf056de5cc3f654ff0bd05d75da4e59a047d559 100644 (file)
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -248,7 +248,8 @@ enum {
 
 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *);
 struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
-                                                 struct request_queue *q);
+                                                 struct request_queue *q,
+                                                 bool elevator_init);
 struct request_queue *blk_mq_init_sq_queue(struct blk_mq_tag_set *set,
                                                const struct blk_mq_ops *ops,
                                                unsigned int queue_depth,