SUNRPC: Micro-optimise when the task is known not to be sleeping

In cases where we know the task is not sleeping, try to optimise
away the indirect call to task->tk_action() by replacing it with
a direct call.
Only change tail calls, to allow gcc to perform tail call
elimination.

Signed-off-by: Trond Myklebust <trond.myklebust@hammerspace.com>

diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h
index 52d41d0c1ae1d54b6829a10134318ee2835b9fd5..ec861cd0cfe8ce9fa5425909d243c660231f06c9 100644 (file)
--- a/include/linux/sunrpc/sched.h
+++ b/include/linux/sunrpc/sched.h
@@ -304,4 +304,12 @@ rpc_clnt_swap_deactivate(struct rpc_clnt *clnt)
 }
 #endif /* CONFIG_SUNRPC_SWAP */
 
+static inline bool
+rpc_task_need_resched(const struct rpc_task *task)
+{
+       if (RPC_IS_QUEUED(task) || task->tk_callback)
+               return true;
+       return false;
+}
+
 #endif /* _LINUX_SUNRPC_SCHED_H_ */

diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index 67c955d8b21b0921c65df34570542a85edd662f7..498dd6ad5bc5799b62cf3aec3ef846c563fea505 100644 (file)
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -1540,6 +1540,7 @@ call_start(struct rpc_task *task)
        clnt->cl_stats->rpccnt++;
        task->tk_action = call_reserve;
        rpc_task_set_transport(task, clnt);
+       call_reserve(task);
 }
 
 /*
@@ -1553,6 +1554,9 @@ call_reserve(struct rpc_task *task)
        task->tk_status  = 0;
        task->tk_action  = call_reserveresult;
        xprt_reserve(task);
+       if (rpc_task_need_resched(task))
+               return;
+        call_reserveresult(task);
 }
 
 static void call_retry_reserve(struct rpc_task *task);
@@ -1575,6 +1579,7 @@ call_reserveresult(struct rpc_task *task)
        if (status >= 0) {
                if (task->tk_rqstp) {
                        task->tk_action = call_refresh;
+                       call_refresh(task);
                        return;
                }
 
@@ -1600,6 +1605,7 @@ call_reserveresult(struct rpc_task *task)
                /* fall through */
        case -EAGAIN:   /* woken up; retry */
                task->tk_action = call_retry_reserve;
+               call_retry_reserve(task);
                return;
        case -EIO:      /* probably a shutdown */
                break;
@@ -1622,6 +1628,9 @@ call_retry_reserve(struct rpc_task *task)
        task->tk_status  = 0;
        task->tk_action  = call_reserveresult;
        xprt_retry_reserve(task);
+       if (rpc_task_need_resched(task))
+               return;
+       call_reserveresult(task);
 }
 
 /*
@@ -1636,6 +1645,9 @@ call_refresh(struct rpc_task *task)
        task->tk_status = 0;
        task->tk_client->cl_stats->rpcauthrefresh++;
        rpcauth_refreshcred(task);
+       if (rpc_task_need_resched(task))
+               return;
+       call_refreshresult(task);
 }
 
 /*
@@ -1654,6 +1666,7 @@ call_refreshresult(struct rpc_task *task)
        case 0:
                if (rpcauth_uptodatecred(task)) {
                        task->tk_action = call_allocate;
+                       call_allocate(task);
                        return;
                }
                /* Use rate-limiting and a max number of retries if refresh
@@ -1672,6 +1685,7 @@ call_refreshresult(struct rpc_task *task)
                task->tk_cred_retry--;
                dprintk("RPC: %5u %s: retry refresh creds\n",
                                task->tk_pid, __func__);
+               call_refresh(task);
                return;
        }
        dprintk("RPC: %5u %s: refresh creds failed with error %d\n",
@@ -1697,8 +1711,10 @@ call_allocate(struct rpc_task *task)
        task->tk_status = 0;
        task->tk_action = call_encode;
 
-       if (req->rq_buffer)
+       if (req->rq_buffer) {
+               call_encode(task);
                return;
+       }
 
        if (proc->p_proc != 0) {
                BUG_ON(proc->p_arglen == 0);
@@ -1719,8 +1735,12 @@ call_allocate(struct rpc_task *task)
 
        status = xprt->ops->buf_alloc(task);
        xprt_inject_disconnect(xprt);
-       if (status == 0)
+       if (status == 0) {
+               if (rpc_task_need_resched(task))
+                       return;
+               call_encode(task);
                return;
+       }
        if (status != -ENOMEM) {
                rpc_exit(task, status);
                return;
@@ -1803,12 +1823,8 @@ call_encode(struct rpc_task *task)
                xprt_request_enqueue_receive(task);
        xprt_request_enqueue_transmit(task);
 out:
-       task->tk_action = call_transmit;
-       /* Check that the connection is OK */
-       if (!xprt_bound(task->tk_xprt))
-               task->tk_action = call_bind;
-       else if (!xprt_connected(task->tk_xprt))
-               task->tk_action = call_connect;
+       task->tk_action = call_bind;
+       call_bind(task);
 }
 
 /*
@@ -1842,14 +1858,17 @@ call_bind(struct rpc_task *task)
                return;
        }
 
+       if (xprt_bound(xprt)) {
+               task->tk_action = call_connect;
+               call_connect(task);
+               return;
+       }
+
        dprint_status(task);
 
-       task->tk_action = call_connect;
-       if (!xprt_bound(xprt)) {
-               task->tk_action = call_bind_status;
-               task->tk_timeout = xprt->bind_timeout;
-               xprt->ops->rpcbind(task);
-       }
+       task->tk_action = call_bind_status;
+       task->tk_timeout = xprt->bind_timeout;
+       xprt->ops->rpcbind(task);
 }
 
 /*
@@ -1869,6 +1888,7 @@ call_bind_status(struct rpc_task *task)
                dprint_status(task);
                task->tk_status = 0;
                task->tk_action = call_connect;
+               call_connect(task);
                return;
        }
 
@@ -1949,21 +1969,24 @@ call_connect(struct rpc_task *task)
                return;
        }
 
+       if (xprt_connected(xprt)) {
+               task->tk_action = call_transmit;
+               call_transmit(task);
+               return;
+       }
+
        dprintk("RPC: %5u call_connect xprt %p %s connected\n",
                        task->tk_pid, xprt,
                        (xprt_connected(xprt) ? "is" : "is not"));
 
-       task->tk_action = call_transmit;
-       if (!xprt_connected(xprt)) {
-               task->tk_action = call_connect_status;
-               if (task->tk_status < 0)
-                       return;
-               if (task->tk_flags & RPC_TASK_NOCONNECT) {
-                       rpc_exit(task, -ENOTCONN);
-                       return;
-               }
-               xprt_connect(task);
+       task->tk_action = call_connect_status;
+       if (task->tk_status < 0)
+               return;
+       if (task->tk_flags & RPC_TASK_NOCONNECT) {
+               rpc_exit(task, -ENOTCONN);
+               return;
        }
+       xprt_connect(task);
 }
 
 /*
@@ -2016,6 +2039,7 @@ call_connect_status(struct rpc_task *task)
        case 0:
                clnt->cl_stats->netreconn++;
                task->tk_action = call_transmit;
+               call_transmit(task);
                return;
        }
        rpc_exit(task, status);
@@ -2040,19 +2064,20 @@ call_transmit(struct rpc_task *task)
        dprint_status(task);
 
        task->tk_action = call_transmit_status;
+       if (!xprt_prepare_transmit(task))
+               return;
+       task->tk_status = 0;
        if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) {
-               if (!xprt_prepare_transmit(task))
+               if (!xprt_connected(task->tk_xprt)) {
+                       task->tk_status = -ENOTCONN;
                        return;
-               task->tk_status = 0;
-               if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) {
-                       if (!xprt_connected(task->tk_xprt)) {
-                               task->tk_status = -ENOTCONN;
-                               return;
-                       }
-                       xprt_transmit(task);
                }
+               xprt_transmit(task);
        }
        xprt_end_transmit(task);
+       if (rpc_task_need_resched(task))
+               return;
+       call_transmit_status(task);
 }
 
 /*
@@ -2067,8 +2092,12 @@ call_transmit_status(struct rpc_task *task)
         * Common case: success.  Force the compiler to put this
         * test first.
         */
-       if (task->tk_status == 0) {
-               xprt_request_wait_receive(task);
+       if (rpc_task_transmitted(task)) {
+               if (task->tk_status == 0)
+                       xprt_request_wait_receive(task);
+               if (rpc_task_need_resched(task))
+                       return;
+               call_status(task);
                return;
        }
 
@@ -2129,6 +2158,7 @@ call_bc_encode(struct rpc_task *task)
 {
        xprt_request_enqueue_transmit(task);
        task->tk_action = call_bc_transmit;
+       call_bc_transmit(task);
 }
 
 /*
@@ -2219,6 +2249,7 @@ call_status(struct rpc_task *task)
        status = task->tk_status;
        if (status >= 0) {
                task->tk_action = call_decode;
+               call_decode(task);
                return;
        }