tcp/udp: Make early_demux back namespacified.

commit c07c8d4a5cce7f22f5f13afe5da94a4bf9e1ba7a upstream.

Commit efd7340c88ac ("ipv4: namespacify ip_early_demux sysctl knob") made
it possible to enable/disable early_demux on a per-netns basis.  Then, we
introduced two knobs, tcp_early_demux and udp_early_demux, to switch it for
TCP/UDP in commit 3271fcf7eb2e ("net: Add sysctl to toggle early demux for
tcp and udp").  However, the .proc_handler() was wrong and actually
disabled us from changing the behaviour in each netns.

We can execute early_demux if net.ipv4.ip_early_demux is on and each proto
.early_demux() handler is not NULL.  When we toggle (tcp|udp)_early_demux,
the change itself is saved in each netns variable, but the .early_demux()
handler is a global variable, so the handler is switched based on the
init_net's sysctl variable.  Thus, netns (tcp|udp)_early_demux knobs have
nothing to do with the logic.  Whether we CAN execute proto .early_demux()
is always decided by init_net's sysctl knob, and whether we DO it or not is
by each netns ip_early_demux knob.

This patch namespacifies (tcp|udp)_early_demux again.  For now, the users
of the .early_demux() handler are TCP and UDP only, and they are called
directly to avoid retpoline.  So, we can remove the .early_demux() handler
from inet6?_protos and need not dereference them in ip6?_rcv_finish_core().
If another proto needs .early_demux(), we can restore it at that time.

Fixes: 3271fcf7eb2e ("net: Add sysctl to toggle early demux for tcp and udp")
Signed-off-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Link: https://lore.kernel.org/r/20220713175207.7727-1-kuniyu@amazon.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/include/net/protocol.h b/include/net/protocol.h
index 2b778e1d2d8f1365c969952b9aed939e28cb80d7..0fd2df844fc71ff81cdf5bf13f9d010a1bbe828d 100644 (file)
--- a/include/net/protocol.h
+++ b/include/net/protocol.h
@@ -35,8 +35,6 @@
 
 /* This is used to register protocols. */
 struct net_protocol {
-       int                     (*early_demux)(struct sk_buff *skb);
-       int                     (*early_demux_handler)(struct sk_buff *skb);
        int                     (*handler)(struct sk_buff *skb);
 
        /* This returns an error if we weren't able to handle the error. */
@@ -53,8 +51,6 @@ struct net_protocol {
 
 #if IS_ENABLED(CONFIG_IPV6)
 struct inet6_protocol {
-       void    (*early_demux)(struct sk_buff *skb);
-       void    (*early_demux_handler)(struct sk_buff *skb);
        int     (*handler)(struct sk_buff *skb);
 
        /* This returns an error if we weren't able to handle the error. */

diff --git a/include/net/tcp.h b/include/net/tcp.h
index 5b2473a08241a8da17c273346304acbb31578374..077feeca6c99e65c4b98fbfbdc44ecc0f88e1993 100644 (file)
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -922,6 +922,8 @@ static inline int tcp_v6_sdif(const struct sk_buff *skb)
 #endif
        return 0;
 }
+
+void tcp_v6_early_demux(struct sk_buff *skb);
 #endif
 
 static inline bool inet_exact_dif_match(struct net *net, struct sk_buff *skb)

diff --git a/include/net/udp.h b/include/net/udp.h
index e66854e767dcc6d62c49ec31c2050f806c13f4a9..bbd607fb939a620199a3f589fd75ed6940b16357 100644 (file)
--- a/include/net/udp.h
+++ b/include/net/udp.h
@@ -169,6 +169,7 @@ typedef struct sock *(*udp_lookup_t)(struct sk_buff *skb, __be16 sport,
 struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
                                struct udphdr *uh, udp_lookup_t lookup);
 int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup);
+void udp_v6_early_demux(struct sk_buff *skb);
 
 struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
                                  netdev_features_t features);

diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index 058dbcb90541e50508f8e2a44d628880dcca5271..3c6412cb4b486870a98d1143bb208a2d4bb395db 100644 (file)
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -1702,12 +1702,7 @@ static const struct net_protocol igmp_protocol = {
 };
 #endif
 
-/* thinking of making this const? Don't.
- * early_demux can change based on sysctl.
- */
-static struct net_protocol tcp_protocol = {
-       .early_demux    =       tcp_v4_early_demux,
-       .early_demux_handler =  tcp_v4_early_demux,
+static const struct net_protocol tcp_protocol = {
        .handler        =       tcp_v4_rcv,
        .err_handler    =       tcp_v4_err,
        .no_policy      =       1,
@@ -1715,12 +1710,7 @@ static struct net_protocol tcp_protocol = {
        .icmp_strict_tag_validation = 1,
 };
 
-/* thinking of making this const? Don't.
- * early_demux can change based on sysctl.
- */
-static struct net_protocol udp_protocol = {
-       .early_demux =  udp_v4_early_demux,
-       .early_demux_handler =  udp_v4_early_demux,
+static const struct net_protocol udp_protocol = {
        .handler =      udp_rcv,
        .err_handler =  udp_err,
        .no_policy =    1,

diff --git a/net/ipv4/ip_input.c b/net/ipv4/ip_input.c
index c59a78a267c37ab3a434c38c0ab236f2f5f2a0f1..1464e273821121ad8bf66d9acee1ce7880f3a2ac 100644 (file)
--- a/net/ipv4/ip_input.c
+++ b/net/ipv4/ip_input.c
@@ -302,31 +302,38 @@ drop:
        return true;
 }
 
-INDIRECT_CALLABLE_DECLARE(int udp_v4_early_demux(struct sk_buff *));
-INDIRECT_CALLABLE_DECLARE(int tcp_v4_early_demux(struct sk_buff *));
+int udp_v4_early_demux(struct sk_buff *);
+int tcp_v4_early_demux(struct sk_buff *);
 static int ip_rcv_finish_core(struct net *net, struct sock *sk,
                              struct sk_buff *skb, struct net_device *dev)
 {
        const struct iphdr *iph = ip_hdr(skb);
-       int (*edemux)(struct sk_buff *skb);
        struct rtable *rt;
        int err;
 
-       if (net->ipv4.sysctl_ip_early_demux &&
+       if (READ_ONCE(net->ipv4.sysctl_ip_early_demux) &&
            !skb_dst(skb) &&
            !skb->sk &&
            !ip_is_fragment(iph)) {
-               const struct net_protocol *ipprot;
-               int protocol = iph->protocol;
-
-               ipprot = rcu_dereference(inet_protos[protocol]);
-               if (ipprot && (edemux = READ_ONCE(ipprot->early_demux))) {
-                       err = INDIRECT_CALL_2(edemux, tcp_v4_early_demux,
-                                             udp_v4_early_demux, skb);
-                       if (unlikely(err))
-                               goto drop_error;
-                       /* must reload iph, skb->head might have changed */
-                       iph = ip_hdr(skb);
+               switch (iph->protocol) {
+               case IPPROTO_TCP:
+                       if (READ_ONCE(net->ipv4.sysctl_tcp_early_demux)) {
+                               tcp_v4_early_demux(skb);
+
+                               /* must reload iph, skb->head might have changed */
+                               iph = ip_hdr(skb);
+                       }
+                       break;
+               case IPPROTO_UDP:
+                       if (READ_ONCE(net->ipv4.sysctl_udp_early_demux)) {
+                               err = udp_v4_early_demux(skb);
+                               if (unlikely(err))
+                                       goto drop_error;
+
+                               /* must reload iph, skb->head might have changed */
+                               iph = ip_hdr(skb);
+                       }
+                       break;
                }
        }
 

diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c
index c83a5d05aeaaf0896846f853c1c6d97b74981726..4d4dba1d42ae150fe316480db7b9c02d7e3985f8 100644 (file)
--- a/net/ipv4/sysctl_net_ipv4.c
+++ b/net/ipv4/sysctl_net_ipv4.c
@@ -363,61 +363,6 @@ bad_key:
        return ret;
 }
 
-static void proc_configure_early_demux(int enabled, int protocol)
-{
-       struct net_protocol *ipprot;
-#if IS_ENABLED(CONFIG_IPV6)
-       struct inet6_protocol *ip6prot;
-#endif
-
-       rcu_read_lock();
-
-       ipprot = rcu_dereference(inet_protos[protocol]);
-       if (ipprot)
-               ipprot->early_demux = enabled ? ipprot->early_demux_handler :
-                                               NULL;
-
-#if IS_ENABLED(CONFIG_IPV6)
-       ip6prot = rcu_dereference(inet6_protos[protocol]);
-       if (ip6prot)
-               ip6prot->early_demux = enabled ? ip6prot->early_demux_handler :
-                                                NULL;
-#endif
-       rcu_read_unlock();
-}
-
-static int proc_tcp_early_demux(struct ctl_table *table, int write,
-                               void __user *buffer, size_t *lenp, loff_t *ppos)
-{
-       int ret = 0;
-
-       ret = proc_dointvec(table, write, buffer, lenp, ppos);
-
-       if (write && !ret) {
-               int enabled = init_net.ipv4.sysctl_tcp_early_demux;
-
-               proc_configure_early_demux(enabled, IPPROTO_TCP);
-       }
-
-       return ret;
-}
-
-static int proc_udp_early_demux(struct ctl_table *table, int write,
-                               void __user *buffer, size_t *lenp, loff_t *ppos)
-{
-       int ret = 0;
-
-       ret = proc_dointvec(table, write, buffer, lenp, ppos);
-
-       if (write && !ret) {
-               int enabled = init_net.ipv4.sysctl_udp_early_demux;
-
-               proc_configure_early_demux(enabled, IPPROTO_UDP);
-       }
-
-       return ret;
-}
-
 static int proc_tfo_blackhole_detect_timeout(struct ctl_table *table,
                                             int write,
                                             void __user *buffer,
@@ -701,14 +646,14 @@ static struct ctl_table ipv4_net_table[] = {
                .data           = &init_net.ipv4.sysctl_udp_early_demux,
                .maxlen         = sizeof(int),
                .mode           = 0644,
-               .proc_handler   = proc_udp_early_demux
+               .proc_handler   = proc_douintvec_minmax,
        },
        {
                .procname       = "tcp_early_demux",
                .data           = &init_net.ipv4.sysctl_tcp_early_demux,
                .maxlen         = sizeof(int),
                .mode           = 0644,
-               .proc_handler   = proc_tcp_early_demux
+               .proc_handler   = proc_douintvec_minmax,
        },
        {
                .procname       = "ip_default_ttl",

diff --git a/net/ipv6/ip6_input.c b/net/ipv6/ip6_input.c
index e6c4966aa956cefc454ad7c52ba72276c043f4b4..ebf90bce063adf0d0872da0064ae6cce24d86002 100644 (file)
--- a/net/ipv6/ip6_input.c
+++ b/net/ipv6/ip6_input.c
@@ -44,21 +44,25 @@
 #include <net/inet_ecn.h>
 #include <net/dst_metadata.h>
 
-INDIRECT_CALLABLE_DECLARE(void udp_v6_early_demux(struct sk_buff *));
-INDIRECT_CALLABLE_DECLARE(void tcp_v6_early_demux(struct sk_buff *));
+void udp_v6_early_demux(struct sk_buff *);
+void tcp_v6_early_demux(struct sk_buff *);
 static void ip6_rcv_finish_core(struct net *net, struct sock *sk,
                                struct sk_buff *skb)
 {
-       void (*edemux)(struct sk_buff *skb);
-
-       if (net->ipv4.sysctl_ip_early_demux && !skb_dst(skb) && skb->sk == NULL) {
-               const struct inet6_protocol *ipprot;
-
-               ipprot = rcu_dereference(inet6_protos[ipv6_hdr(skb)->nexthdr]);
-               if (ipprot && (edemux = READ_ONCE(ipprot->early_demux)))
-                       INDIRECT_CALL_2(edemux, tcp_v6_early_demux,
-                                       udp_v6_early_demux, skb);
+       if (READ_ONCE(net->ipv4.sysctl_ip_early_demux) &&
+           !skb_dst(skb) && !skb->sk) {
+               switch (ipv6_hdr(skb)->nexthdr) {
+               case IPPROTO_TCP:
+                       if (READ_ONCE(net->ipv4.sysctl_tcp_early_demux))
+                               tcp_v6_early_demux(skb);
+                       break;
+               case IPPROTO_UDP:
+                       if (READ_ONCE(net->ipv4.sysctl_udp_early_demux))
+                               udp_v6_early_demux(skb);
+                       break;
+               }
        }
+
        if (!skb_valid_dst(skb))
                ip6_route_input(skb);
 }

diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index 397c4597c438988cd00432969bc9258b55de6e99..831f779aba7b02207d1ef6128924c0c69646dfad 100644 (file)
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -1729,7 +1729,7 @@ do_time_wait:
        goto discard_it;
 }
 
-INDIRECT_CALLABLE_SCOPE void tcp_v6_early_demux(struct sk_buff *skb)
+void tcp_v6_early_demux(struct sk_buff *skb)
 {
        const struct ipv6hdr *hdr;
        const struct tcphdr *th;
@@ -2084,12 +2084,7 @@ struct proto tcpv6_prot = {
        .diag_destroy           = tcp_abort,
 };
 
-/* thinking of making this const? Don't.
- * early_demux can change based on sysctl.
- */
-static struct inet6_protocol tcpv6_protocol = {
-       .early_demux    =       tcp_v6_early_demux,
-       .early_demux_handler =  tcp_v6_early_demux,
+static const struct inet6_protocol tcpv6_protocol = {
        .handler        =       tcp_v6_rcv,
        .err_handler    =       tcp_v6_err,
        .flags          =       INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,

diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 62c0db6df56304dd3848e56f50e3f0da2fb019c3..fd1ce0405b7ea10a09533bffb04cc8277073d2ea 100644 (file)
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -973,7 +973,7 @@ static struct sock *__udp6_lib_demux_lookup(struct net *net,
        return NULL;
 }
 
-INDIRECT_CALLABLE_SCOPE void udp_v6_early_demux(struct sk_buff *skb)
+void udp_v6_early_demux(struct sk_buff *skb)
 {
        struct net *net = dev_net(skb->dev);
        const struct udphdr *uh;
@@ -1603,12 +1603,7 @@ int compat_udpv6_getsockopt(struct sock *sk, int level, int optname,
 }
 #endif
 
-/* thinking of making this const? Don't.
- * early_demux can change based on sysctl.
- */
-static struct inet6_protocol udpv6_protocol = {
-       .early_demux    =       udp_v6_early_demux,
-       .early_demux_handler =  udp_v6_early_demux,
+static const struct inet6_protocol udpv6_protocol = {
        .handler        =       udpv6_rcv,
        .err_handler    =       udpv6_err,
        .flags          =       INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,