io-wq: support concurrent non-blocking work

io-wq assumes that work will complete fast (and not block), so it
doesn't create a new worker when work is enqueued, if we already have
at least one worker running. This is done on the assumption that if work
is running, then it will complete fast.

Add an option to force io-wq to fork a new worker for work queued. This
is signaled by setting IO_WQ_WORK_CONCURRENT on the work item. For that
case, io-wq will create a new worker, even though workers are already
running.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io-wq.c b/fs/io-wq.c
index 79eae29983caf5b63c22275756883dbbbd1397cb..4d902c19ee5f3ce064d554a49e7ee5460282ce45 100644 (file)
--- a/fs/io-wq.c
+++ b/fs/io-wq.c
@@ -724,6 +724,7 @@ static bool io_wq_can_queue(struct io_wqe *wqe, struct io_wqe_acct *acct,
 static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work)
 {
        struct io_wqe_acct *acct = io_work_get_acct(wqe, work);
+       int work_flags;
        unsigned long flags;
 
        /*
@@ -738,12 +739,14 @@ static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work)
                return;
        }
 
+       work_flags = work->flags;
        spin_lock_irqsave(&wqe->lock, flags);
        wq_list_add_tail(&work->list, &wqe->work_list);
        wqe->flags &= ~IO_WQE_FLAG_STALLED;
        spin_unlock_irqrestore(&wqe->lock, flags);
 
-       if (!atomic_read(&acct->nr_running))
+       if ((work_flags & IO_WQ_WORK_CONCURRENT) ||
+           !atomic_read(&acct->nr_running))
                io_wqe_wake_worker(wqe, acct);
 }
 

diff --git a/fs/io-wq.h b/fs/io-wq.h
index 04d60ad38dfc9db3d14f563dd58382ff28e614cb..1cd039af88134aa3163ebfa0320a05ab43543db7 100644 (file)
--- a/fs/io-wq.h
+++ b/fs/io-wq.h
@@ -13,6 +13,7 @@ enum {
        IO_WQ_WORK_INTERNAL     = 64,
        IO_WQ_WORK_CB           = 128,
        IO_WQ_WORK_NO_CANCEL    = 256,
+       IO_WQ_WORK_CONCURRENT   = 512,
 
        IO_WQ_HASH_SHIFT        = 24,   /* upper 8 bits are used for hash key */
 };