blk-mq: complete polled requests directly

Even for single queue devices there is no point in offloading a polled
completion to the softirq, given that blk_mq_force_complete_rq is called
from the polling thread in that case and thus there are no starvation
issues.

Reviewed-by: Daniel Wagner <dwagner@suse.de>
Signed-off-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-mq.c b/block/blk-mq.c
index ada55521601f9a94aaea754aeb793bd3724390af..ea083f58d9da550a9f9f1179a6be4ac9cc24eee3 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -703,6 +703,16 @@ void blk_mq_force_complete_rq(struct request *rq)
        int cpu;
 
        WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
+
+       /*
+        * For a polled request, always complete locallly, it's pointless
+        * to redirect the completion.
+        */
+       if (rq->cmd_flags & REQ_HIPRI) {
+               q->mq_ops->complete(rq);
+               return;
+       }
+
        /*
         * Most of single queue controllers, there is only one irq vector
         * for handling IO completion, and the only irq's affinity is set
@@ -717,12 +727,7 @@ void blk_mq_force_complete_rq(struct request *rq)
                return;
        }
 
-       /*
-        * For a polled request, always complete locallly, it's pointless
-        * to redirect the completion.
-        */
-       if ((rq->cmd_flags & REQ_HIPRI) ||
-           !test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags)) {
+       if (!test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags)) {
                q->mq_ops->complete(rq);
                return;
        }