io_uring: prevent potential eventfd recursion on poll

If we have nested or circular eventfd wakeups, then we can deadlock if
we run them inline from our poll waitqueue wakeup handler. It's also
possible to have very long chains of notifications, to the extent where
we could risk blowing the stack.

Check the eventfd recursion count before calling eventfd_signal(). If
it's non-zero, then punt the signaling to async context. This is always
safe, as it takes us out-of-line in terms of stack and locking context.

Cc: stable@vger.kernel.org # 5.1+
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 217721c7bc41ede3f30d5a83f391fc50af731d31..43f3b7d90299756eec5a97486e60ba090fc4cfe7 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -1020,21 +1020,28 @@ static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
 
 static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
 {
+       if (!ctx->cq_ev_fd)
+               return false;
        if (!ctx->eventfd_async)
                return true;
        return io_wq_current_is_worker() || in_interrupt();
 }
 
-static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
+static void __io_cqring_ev_posted(struct io_ring_ctx *ctx, bool trigger_ev)
 {
        if (waitqueue_active(&ctx->wait))
                wake_up(&ctx->wait);
        if (waitqueue_active(&ctx->sqo_wait))
                wake_up(&ctx->sqo_wait);
-       if (ctx->cq_ev_fd && io_should_trigger_evfd(ctx))
+       if (trigger_ev)
                eventfd_signal(ctx->cq_ev_fd, 1);
 }
 
+static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
+{
+       __io_cqring_ev_posted(ctx, io_should_trigger_evfd(ctx));
+}
+
 /* Returns true if there are no backlogged entries after the flush */
 static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
 {
@@ -3561,6 +3568,14 @@ static void io_poll_flush(struct io_wq_work **workptr)
                __io_poll_flush(req->ctx, nodes);
 }
 
+static void io_poll_trigger_evfd(struct io_wq_work **workptr)
+{
+       struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
+
+       eventfd_signal(req->ctx->cq_ev_fd, 1);
+       io_put_req(req);
+}
+
 static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
                        void *key)
 {
@@ -3586,14 +3601,22 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
 
                if (llist_empty(&ctx->poll_llist) &&
                    spin_trylock_irqsave(&ctx->completion_lock, flags)) {
+                       bool trigger_ev;
+
                        hash_del(&req->hash_node);
                        io_poll_complete(req, mask, 0);
-                       req->flags |= REQ_F_COMP_LOCKED;
-                       io_put_req(req);
-                       spin_unlock_irqrestore(&ctx->completion_lock, flags);
 
-                       io_cqring_ev_posted(ctx);
-                       req = NULL;
+                       trigger_ev = io_should_trigger_evfd(ctx);
+                       if (trigger_ev && eventfd_signal_count()) {
+                               trigger_ev = false;
+                               req->work.func = io_poll_trigger_evfd;
+                       } else {
+                               req->flags |= REQ_F_COMP_LOCKED;
+                               io_put_req(req);
+                               req = NULL;
+                       }
+                       spin_unlock_irqrestore(&ctx->completion_lock, flags);
+                       __io_cqring_ev_posted(ctx, trigger_ev);
                } else {
                        req->result = mask;
                        req->llist_node.next = NULL;