xsk, net: Make sock_def_readable() have external linkage

XDP sockets use the default implementation of struct sock's
sk_data_ready callback, which is sock_def_readable(). This function
is called in the XDP socket fast-path, and involves a retpoline. By
letting sock_def_readable() have external linkage, and being called
directly, the retpoline can be avoided.

Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20200120092917.13949-1-bjorn.topel@gmail.com

diff --git a/include/net/sock.h b/include/net/sock.h
index 8dff68b4c3168a75c762e31e8fe91abb519554e2..0891c55f1e824d697271fcf31c351f7252083a8a 100644 (file)
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -2612,4 +2612,6 @@ static inline bool sk_dev_equal_l3scope(struct sock *sk, int dif)
        return false;
 }
 
+void sock_def_readable(struct sock *sk);
+
 #endif /* _SOCK_H */

diff --git a/net/core/sock.c b/net/core/sock.c
index 8459ad579f735ce724b559f7114d1b77f360e5b2..a4c8fac781ff3ceba7fb6c85f636e18a0de66891 100644 (file)
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -2786,7 +2786,7 @@ static void sock_def_error_report(struct sock *sk)
        rcu_read_unlock();
 }
 
-static void sock_def_readable(struct sock *sk)
+void sock_def_readable(struct sock *sk)
 {
        struct socket_wq *wq;
 

diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 02ada7ab8c6eb08a0e2ca56353dd0dc62dd55911..df600487a68d513c3f7abcb1b584d2f4c229a7dd 100644 (file)
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -217,7 +217,7 @@ static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
 static void xsk_flush(struct xdp_sock *xs)
 {
        xskq_prod_submit(xs->rx);
-       xs->sk.sk_data_ready(&xs->sk);
+       sock_def_readable(&xs->sk);
 }
 
 int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)