txhash: Add socket option to control TX hash rethink behavior

Add the SO_TXREHASH socket option to control hash rethink behavior per socket.
When default mode is set, sockets disable rehash at initialization and use
sysctl option when entering listen state. setsockopt() overrides default
behavior.

Signed-off-by: Akhmat Karakotov <hmukos@yandex-team.ru>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/arch/alpha/include/uapi/asm/socket.h b/arch/alpha/include/uapi/asm/socket.h
index 284d28755b8de6c4ae9278710db152e9121ad267..7d81535893af6195d43319954a4e7c69bd4738c2 100644 (file)
--- a/arch/alpha/include/uapi/asm/socket.h
+++ b/arch/alpha/include/uapi/asm/socket.h
@@ -133,6 +133,8 @@
 
 #define SO_RESERVE_MEM         73
 
+#define SO_TXREHASH            74
+
 #if !defined(__KERNEL__)
 
 #if __BITS_PER_LONG == 64

diff --git a/arch/mips/include/uapi/asm/socket.h b/arch/mips/include/uapi/asm/socket.h
index 24e0efb360f6729f4fee8a146e5da3a751ec6667..1d55e57b846663e7c762d30b366722d3cf4a1393 100644 (file)
--- a/arch/mips/include/uapi/asm/socket.h
+++ b/arch/mips/include/uapi/asm/socket.h
@@ -144,6 +144,8 @@
 
 #define SO_RESERVE_MEM         73
 
+#define SO_TXREHASH            74
+
 #if !defined(__KERNEL__)
 
 #if __BITS_PER_LONG == 64

diff --git a/arch/parisc/include/uapi/asm/socket.h b/arch/parisc/include/uapi/asm/socket.h
index 845ddc63c88227db264f454559fdaefb13d72222..654061e0964edfa53cba57d4d5f4e50a8f19b5b1 100644 (file)
--- a/arch/parisc/include/uapi/asm/socket.h
+++ b/arch/parisc/include/uapi/asm/socket.h
@@ -125,6 +125,8 @@
 
 #define SO_RESERVE_MEM         0x4047
 
+#define SO_TXREHASH            0x4048
+
 #if !defined(__KERNEL__)
 
 #if __BITS_PER_LONG == 64

diff --git a/arch/sparc/include/uapi/asm/socket.h b/arch/sparc/include/uapi/asm/socket.h
index 2672dd03faf311ad751f93145646ebe441055513..666f81e617eaca90df756c3e192b4bef07fe8057 100644 (file)
--- a/arch/sparc/include/uapi/asm/socket.h
+++ b/arch/sparc/include/uapi/asm/socket.h
@@ -126,6 +126,8 @@
 
 #define SO_RESERVE_MEM           0x0052
 
+#define SO_TXREHASH              0x0053
+
 
 #if !defined(__KERNEL__)
 

diff --git a/include/net/sock.h b/include/net/sock.h
index 0540e1b2aeb0ffe46acf1bb8f2bc8794e53229b7..d6c13f0fba4018541cb663ac1ebfdec024275a24 100644 (file)
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -316,6 +316,7 @@ struct sk_filter;
   *    @sk_rcvtimeo: %SO_RCVTIMEO setting
   *    @sk_sndtimeo: %SO_SNDTIMEO setting
   *    @sk_txhash: computed flow hash for use on transmit
+  *    @sk_txrehash: enable TX hash rethink
   *    @sk_filter: socket filtering instructions
   *    @sk_timer: sock cleanup timer
   *    @sk_stamp: time stamp of last packet received
@@ -491,6 +492,7 @@ struct sock {
        u32                     sk_ack_backlog;
        u32                     sk_max_ack_backlog;
        kuid_t                  sk_uid;
+       u8                      sk_txrehash;
 #ifdef CONFIG_NET_RX_BUSY_POLL
        u8                      sk_prefer_busy_poll;
        u16                     sk_busy_poll_budget;
@@ -2066,18 +2068,10 @@ static inline void sk_set_txhash(struct sock *sk)
 
 static inline bool sk_rethink_txhash(struct sock *sk)
 {
-       u8 rehash;
-
-       if (!sk->sk_txhash)
-               return false;
-
-       rehash = READ_ONCE(sock_net(sk)->core.sysctl_txrehash);
-
-       if (rehash) {
+       if (sk->sk_txhash && sk->sk_txrehash == SOCK_TXREHASH_ENABLED) {
                sk_set_txhash(sk);
                return true;
        }
-
        return false;
 }
 

diff --git a/include/uapi/asm-generic/socket.h b/include/uapi/asm-generic/socket.h
index c77a1313b3b0be60c0a83342ad7880158c13c6db..467ca2f28760e07f80e29600c204c8a5da6c2882 100644 (file)
--- a/include/uapi/asm-generic/socket.h
+++ b/include/uapi/asm-generic/socket.h
@@ -128,6 +128,8 @@
 
 #define SO_RESERVE_MEM         73
 
+#define SO_TXREHASH            74
+
 #if !defined(__KERNEL__)
 
 #if __BITS_PER_LONG == 64 || (defined(__x86_64__) && defined(__ILP32__))

diff --git a/include/uapi/linux/socket.h b/include/uapi/linux/socket.h
index 0accd6102ece5223502f0d8501e8994fcb69e715..51d6bb2f67656af7e3a124542a319c5eed92330b 100644 (file)
--- a/include/uapi/linux/socket.h
+++ b/include/uapi/linux/socket.h
@@ -31,6 +31,7 @@ struct __kernel_sockaddr_storage {
 
 #define SOCK_BUF_LOCK_MASK (SOCK_SNDBUF_LOCK | SOCK_RCVBUF_LOCK)
 
+#define SOCK_TXREHASH_DEFAULT  ((u8)-1)
 #define SOCK_TXREHASH_DISABLED 0
 #define SOCK_TXREHASH_ENABLED  1
 

diff --git a/net/core/sock.c b/net/core/sock.c
index cccf21f3618dfbe8bbc413863e504c8d3096883f..5e711b42898fcaa8e5d60e909d6accf67902def3 100644 (file)
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1447,6 +1447,14 @@ set_sndbuf:
                break;
        }
 
+       case SO_TXREHASH:
+               if (val < -1 || val > 1) {
+                       ret = -EINVAL;
+                       break;
+               }
+               sk->sk_txrehash = (u8)val;
+               break;
+
        default:
                ret = -ENOPROTOOPT;
                break;
@@ -1834,6 +1842,10 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
                v.val = sk->sk_reserved_mem;
                break;
 
+       case SO_TXREHASH:
+               v.val = sk->sk_txrehash;
+               break;
+
        default:
                /* We implement the SO_SNDLOWAT etc to not be settable
                 * (1003.1g 7).
@@ -3279,6 +3291,7 @@ void sock_init_data(struct socket *sock, struct sock *sk)
        sk->sk_pacing_rate = ~0UL;
        WRITE_ONCE(sk->sk_pacing_shift, 10);
        sk->sk_incoming_cpu = -1;
+       sk->sk_txrehash = SOCK_TXREHASH_DEFAULT;
 
        sk_rx_queue_clear(sk);
        /*

diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c
index fc2a985f6064246f522ab146e9d15d9c36ab5072..b81fb13fc5f446e065d00d5bbb2555a91d635dbd 100644 (file)
--- a/net/ipv4/inet_connection_sock.c
+++ b/net/ipv4/inet_connection_sock.c
@@ -1046,6 +1046,9 @@ int inet_csk_listen_start(struct sock *sk)
        sk->sk_ack_backlog = 0;
        inet_csk_delack_init(sk);
 
+       if (sk->sk_txrehash == SOCK_TXREHASH_DEFAULT)
+               sk->sk_txrehash = READ_ONCE(sock_net(sk)->core.sysctl_txrehash);
+
        /* There is race window here: we announce ourselves listening,
         * but this transition is still not validated by get_port().
         * It is OK, because this socket enters to hash table only