tcp: avoid indirect calls to sock_rfree

TCP uses sk_eat_skb() when skbs can be removed from receive queue.
However, the call to skb_orphan() from __kfree_skb() incurs
an indirect call so sock_rfee(), which is more expensive than
a direct call, especially for CONFIG_RETPOLINE=y.

Add tcp_eat_recv_skb() function to make the call before
__kfree_skb().

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 9175e0d729f5e65b5fa39acadc5bf9de715854ad..4e7011672aa9a04370b7a03b972fe19cd48ea232 100644 (file)
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -1580,6 +1580,16 @@ void tcp_cleanup_rbuf(struct sock *sk, int copied)
                tcp_send_ack(sk);
 }
 
+static void tcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb)
+{
+       if (likely(skb->destructor == sock_rfree)) {
+               sock_rfree(skb);
+               skb->destructor = NULL;
+               skb->sk = NULL;
+       }
+       sk_eat_skb(sk, skb);
+}
+
 static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
 {
        struct sk_buff *skb;
@@ -1599,7 +1609,7 @@ static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
                 * splitted a fat GRO packet, while we released socket lock
                 * in skb_splice_bits()
                 */
-               sk_eat_skb(sk, skb);
+               tcp_eat_recv_skb(sk, skb);
        }
        return NULL;
 }
@@ -1665,11 +1675,11 @@ int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
                                continue;
                }
                if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
-                       sk_eat_skb(sk, skb);
+                       tcp_eat_recv_skb(sk, skb);
                        ++seq;
                        break;
                }
-               sk_eat_skb(sk, skb);
+               tcp_eat_recv_skb(sk, skb);
                if (!desc->count)
                        break;
                WRITE_ONCE(tp->copied_seq, seq);
@@ -2481,14 +2491,14 @@ skip_copy:
                if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
                        goto found_fin_ok;
                if (!(flags & MSG_PEEK))
-                       sk_eat_skb(sk, skb);
+                       tcp_eat_recv_skb(sk, skb);
                continue;
 
 found_fin_ok:
                /* Process the FIN. */
                WRITE_ONCE(*seq, *seq + 1);
                if (!(flags & MSG_PEEK))
-                       sk_eat_skb(sk, skb);
+                       tcp_eat_recv_skb(sk, skb);
                break;
        } while (len > 0);