xprtrdma: Invoke rpcrdma_ia_open in the connect worker

Move rdma_cm_id creation into rpcrdma_ep_create() so that it is now
responsible for allocating all per-connection hardware resources.

With this clean-up, all three arms of the switch statement in
rpcrdma_ep_connect are exactly the same now, thus the switch can be
removed.

Because device removal behaves a little differently than
disconnection, there is a little more work to be done before
rpcrdma_ep_destroy() can release the connection's rdma_cm_id. So
it is not quite symmetrical with rpcrdma_ep_create() yet.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>

diff --git a/include/trace/events/rpcrdma.h b/include/trace/events/rpcrdma.h
index 87f4461ab108a6486f2f6f610cac2c4815a23bf1..ba37c47b51e8288583ccc4fbc0196a4c80e98eb9 100644 (file)
--- a/include/trace/events/rpcrdma.h
+++ b/include/trace/events/rpcrdma.h
@@ -415,7 +415,6 @@ DEFINE_CONN_EVENT(disconnect);
 DEFINE_RXPRT_EVENT(xprtrdma_create);
 DEFINE_RXPRT_EVENT(xprtrdma_op_destroy);
 DEFINE_RXPRT_EVENT(xprtrdma_remove);
-DEFINE_RXPRT_EVENT(xprtrdma_reinsert);
 DEFINE_RXPRT_EVENT(xprtrdma_op_inject_dsc);
 DEFINE_RXPRT_EVENT(xprtrdma_op_close);
 DEFINE_RXPRT_EVENT(xprtrdma_op_setport);

diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c
index 6349e6c98b57d6909f2906ba07159bbc45893369..745dfd14963734b9545fe945435ee99b394daefb 100644 (file)
--- a/net/sunrpc/xprtrdma/transport.c
+++ b/net/sunrpc/xprtrdma/transport.c
@@ -286,7 +286,6 @@ xprt_rdma_destroy(struct rpc_xprt *xprt)
 
        rpcrdma_xprt_disconnect(r_xprt);
        rpcrdma_buffer_destroy(&r_xprt->rx_buf);
-       rpcrdma_ia_close(&r_xprt->rx_ia);
 
        xprt_rdma_free_addresses(xprt);
        xprt_free(xprt);
@@ -347,10 +346,6 @@ xprt_setup_rdma(struct xprt_create *args)
        xprt_rdma_format_addresses(xprt, sap);
 
        new_xprt = rpcx_to_rdmax(xprt);
-       rc = rpcrdma_ia_open(new_xprt);
-       if (rc)
-               goto out1;
-
        rc = rpcrdma_buffer_create(new_xprt);
        if (rc)
                goto out2;
@@ -372,8 +367,6 @@ out4:
        rpcrdma_buffer_destroy(&new_xprt->rx_buf);
        rc = -ENODEV;
 out2:
-       rpcrdma_ia_close(&new_xprt->rx_ia);
-out1:
        trace_xprtrdma_op_destroy(new_xprt);
        xprt_rdma_free_addresses(xprt);
        xprt_free(xprt);

diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index 36fe7baea01435a3793131d9aa6b5b110308a767..3df20f3555793510c88aa3e4c2838a0f6bbc83c1 100644 (file)
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -345,31 +345,6 @@ out:
  * Exported functions.
  */
 
-/**
- * rpcrdma_ia_open - Open and initialize an Interface Adapter.
- * @xprt: transport with IA to (re)initialize
- *
- * Returns 0 on success, negative errno if an appropriate
- * Interface Adapter could not be found and opened.
- */
-int
-rpcrdma_ia_open(struct rpcrdma_xprt *xprt)
-{
-       struct rpcrdma_ia *ia = &xprt->rx_ia;
-       int rc;
-
-       ia->ri_id = rpcrdma_create_id(xprt, ia);
-       if (IS_ERR(ia->ri_id)) {
-               rc = PTR_ERR(ia->ri_id);
-               goto out_err;
-       }
-       return 0;
-
-out_err:
-       rpcrdma_ia_close(ia);
-       return rc;
-}
-
 /**
  * rpcrdma_ia_remove - Handle device driver unload
  * @ia: interface adapter being removed
@@ -401,34 +376,26 @@ rpcrdma_ia_remove(struct rpcrdma_ia *ia)
        trace_xprtrdma_remove(r_xprt);
 }
 
-/**
- * rpcrdma_ia_close - Clean up/close an IA.
- * @ia: interface adapter to close
- *
- */
-void
-rpcrdma_ia_close(struct rpcrdma_ia *ia)
-{
-       if (ia->ri_id && !IS_ERR(ia->ri_id))
-               rdma_destroy_id(ia->ri_id);
-       ia->ri_id = NULL;
-}
-
-static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt,
-                            struct rdma_cm_id *id)
+static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt)
 {
        struct rpcrdma_ep *ep = &r_xprt->rx_ep;
        struct rpcrdma_ia *ia = &r_xprt->rx_ia;
        struct rpcrdma_connect_private *pmsg = &ep->rep_cm_private;
+       struct rdma_cm_id *id;
        int rc;
 
+       id = rpcrdma_create_id(r_xprt, ia);
+       if (IS_ERR(id))
+               return PTR_ERR(id);
+
        ep->rep_max_requests = r_xprt->rx_xprt.max_reqs;
        ep->rep_inline_send = xprt_rdma_max_inline_write;
        ep->rep_inline_recv = xprt_rdma_max_inline_read;
 
        rc = frwr_query_device(r_xprt, id->device);
        if (rc)
-               return rc;
+               goto out_destroy;
+
        r_xprt->rx_buf.rb_max_requests = cpu_to_be32(ep->rep_max_requests);
 
        ep->rep_attr.event_handler = rpcrdma_qp_event_handler;
@@ -507,10 +474,12 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt,
        rc = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr);
        if (rc)
                goto out_destroy;
+       ia->ri_id = id;
        return 0;
 
 out_destroy:
        rpcrdma_ep_destroy(r_xprt);
+       rdma_destroy_id(id);
        return rc;
 }
 
@@ -536,79 +505,8 @@ static void rpcrdma_ep_destroy(struct rpcrdma_xprt *r_xprt)
        ia->ri_pd = NULL;
 }
 
-/* Re-establish a connection after a device removal event.
- * Unlike a normal reconnection, a fresh PD and a new set
- * of MRs and buffers is needed.
- */
-static int rpcrdma_ep_recreate_xprt(struct rpcrdma_xprt *r_xprt)
-{
-       struct rpcrdma_ia *ia = &r_xprt->rx_ia;
-       int rc, err;
-
-       trace_xprtrdma_reinsert(r_xprt);
-
-       rc = -EHOSTUNREACH;
-       if (rpcrdma_ia_open(r_xprt))
-               goto out1;
-
-       rc = -ENETUNREACH;
-       err = rpcrdma_ep_create(r_xprt, ia->ri_id);
-       if (err)
-               goto out2;
-       return 0;
-
-out2:
-       rpcrdma_ia_close(ia);
-out1:
-       return rc;
-}
-
-static int rpcrdma_ep_reconnect(struct rpcrdma_xprt *r_xprt)
-{
-       struct rpcrdma_ia *ia = &r_xprt->rx_ia;
-       struct rdma_cm_id *id, *old;
-       int err, rc;
-
-       rc = -EHOSTUNREACH;
-       id = rpcrdma_create_id(r_xprt, ia);
-       if (IS_ERR(id))
-               goto out;
-
-       /* As long as the new ID points to the same device as the
-        * old ID, we can reuse the transport's existing PD and all
-        * previously allocated MRs. Also, the same device means
-        * the transport's previous DMA mappings are still valid.
-        *
-        * This is a sanity check only. There should be no way these
-        * point to two different devices here.
-        */
-       old = id;
-       rc = -ENETUNREACH;
-       if (ia->ri_id->device != id->device) {
-               pr_err("rpcrdma: can't reconnect on different device!\n");
-               goto out_destroy;
-       }
-
-       err = rpcrdma_ep_create(r_xprt, id);
-       if (err)
-               goto out_destroy;
-
-       /* Atomically replace the transport's ID. */
-       rc = 0;
-       old = ia->ri_id;
-       ia->ri_id = id;
-
-out_destroy:
-       rdma_destroy_id(old);
-out:
-       return rc;
-}
-
-/**
- * rpcrdma_xprt_connect - Connect an unconnected transport
- * @r_xprt: controlling transport instance
- *
- * Returns 0 on success or a negative errno.
+/*
+ * Connect unconnected endpoint.
  */
 int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt)
 {
@@ -618,25 +516,10 @@ int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt)
        int rc;
 
 retry:
-       switch (ep->rep_connected) {
-       case 0:
-               rc = -ENETUNREACH;
-               if (rpcrdma_ep_create(r_xprt, ia->ri_id))
-                       goto out_noupdate;
-               break;
-       case -ENODEV:
-               rc = rpcrdma_ep_recreate_xprt(r_xprt);
-               if (rc)
-                       goto out_noupdate;
-               break;
-       case 1:
-               rpcrdma_xprt_disconnect(r_xprt);
-               /* fall through */
-       default:
-               rc = rpcrdma_ep_reconnect(r_xprt);
-               if (rc)
-                       goto out;
-       }
+       rpcrdma_xprt_disconnect(r_xprt);
+       rc = rpcrdma_ep_create(r_xprt);
+       if (rc)
+               goto out_noupdate;
 
        ep->rep_connected = 0;
        xprt_clear_connected(xprt);
@@ -712,6 +595,10 @@ out:
        rpcrdma_sendctxs_destroy(r_xprt);
 
        rpcrdma_ep_destroy(r_xprt);
+
+       if (ia->ri_id)
+               rdma_destroy_id(ia->ri_id);
+       ia->ri_id = NULL;
 }
 
 /* Fixed-size circular FIFO queue. This implementation is wait-free and

diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index 9ead06b1d8a45d50855dffefc7dfc309ddd14886..8be1b70b71a2c7d9fd521d8416e9099a0f6dad9a 100644 (file)
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -457,9 +457,7 @@ extern unsigned int xprt_rdma_memreg_strategy;
 /*
  * Interface Adapter calls - xprtrdma/verbs.c
  */
-int rpcrdma_ia_open(struct rpcrdma_xprt *xprt);
 void rpcrdma_ia_remove(struct rpcrdma_ia *ia);
-void rpcrdma_ia_close(struct rpcrdma_ia *);
 
 /*
  * Endpoint calls - xprtrdma/verbs.c