io_uring: extract a helper for ctx quiesce

Refactor __io_uring_register() by extracting a helper responsible for
ctx queisce. Looks better and will make it easier to add more
optimisations.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/0339e0027504176be09237eefa7945bf9a6f153d.1628471125.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 14aaeb87b149bf0c54d01651ae9341cb391d3a77..03c32d392735e0f78fa26ca8a4a6bc1fb735f512 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -10074,6 +10074,33 @@ static bool io_register_op_must_quiesce(int op)
        }
 }
 
+static int io_ctx_quiesce(struct io_ring_ctx *ctx)
+{
+       long ret;
+
+       percpu_ref_kill(&ctx->refs);
+
+       /*
+        * Drop uring mutex before waiting for references to exit. If another
+        * thread is currently inside io_uring_enter() it might need to grab the
+        * uring_lock to make progress. If we hold it here across the drain
+        * wait, then we can deadlock. It's safe to drop the mutex here, since
+        * no new references will come in after we've killed the percpu ref.
+        */
+       mutex_unlock(&ctx->uring_lock);
+       do {
+               ret = wait_for_completion_interruptible(&ctx->ref_comp);
+               if (!ret)
+                       break;
+               ret = io_run_task_work_sig();
+       } while (ret >= 0);
+       mutex_lock(&ctx->uring_lock);
+
+       if (ret)
+               io_refs_resurrect(&ctx->refs, &ctx->ref_comp);
+       return ret;
+}
+
 static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
                               void __user *arg, unsigned nr_args)
        __releases(ctx->uring_lock)
@@ -10098,31 +10125,9 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
        }
 
        if (io_register_op_must_quiesce(opcode)) {
-               percpu_ref_kill(&ctx->refs);
-
-               /*
-                * Drop uring mutex before waiting for references to exit. If
-                * another thread is currently inside io_uring_enter() it might
-                * need to grab the uring_lock to make progress. If we hold it
-                * here across the drain wait, then we can deadlock. It's safe
-                * to drop the mutex here, since no new references will come in
-                * after we've killed the percpu ref.
-                */
-               mutex_unlock(&ctx->uring_lock);
-               do {
-                       ret = wait_for_completion_interruptible(&ctx->ref_comp);
-                       if (!ret)
-                               break;
-                       ret = io_run_task_work_sig();
-                       if (ret < 0)
-                               break;
-               } while (1);
-               mutex_lock(&ctx->uring_lock);
-
-               if (ret) {
-                       io_refs_resurrect(&ctx->refs, &ctx->ref_comp);
+               ret = io_ctx_quiesce(ctx);
+               if (ret)
                        return ret;
-               }
        }
 
        switch (opcode) {