neighbour: annotate lockless accesses to n->nud_state

[ Upstream commit b071af523579df7341cabf0f16fc661125e9a13f ]

We have many lockless accesses to n->nud_state.

Before adding another one in the following patch,
add annotations to readers and writers.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: David Ahern <dsahern@kernel.org>
Reviewed-by: Martin KaFai Lau <martin.lau@kernel.org>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
Stable-dep-of: 5baa0433a15e ("neighbour: fix data-races around n->output")
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c
index 0c3eb850fcb7978adac5ab03f33d4a63741f6282..619dd71c9d75eab33ab4482bc4a898433d438cd8 100644 (file)
--- a/drivers/net/vxlan/vxlan_core.c
+++ b/drivers/net/vxlan/vxlan_core.c
@@ -1910,7 +1910,7 @@ static int arp_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni)
                struct vxlan_fdb *f;
                struct sk_buff  *reply;
 
-               if (!(n->nud_state & NUD_CONNECTED)) {
+               if (!(READ_ONCE(n->nud_state) & NUD_CONNECTED)) {
                        neigh_release(n);
                        goto out;
                }
@@ -2074,7 +2074,7 @@ static int neigh_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni)
                struct vxlan_fdb *f;
                struct sk_buff *reply;
 
-               if (!(n->nud_state & NUD_CONNECTED)) {
+               if (!(READ_ONCE(n->nud_state) & NUD_CONNECTED)) {
                        neigh_release(n);
                        goto out;
                }

diff --git a/include/net/neighbour.h b/include/net/neighbour.h
index 794e45981891a60b4aefd6b6551f2a46e4dfc599..cd30aac9ec835b97b9debe03e259ae7c0ecc44c8 100644 (file)
--- a/include/net/neighbour.h
+++ b/include/net/neighbour.h
@@ -464,7 +464,7 @@ static __always_inline int neigh_event_send_probe(struct neighbour *neigh,
 
        if (READ_ONCE(neigh->used) != now)
                WRITE_ONCE(neigh->used, now);
-       if (!(neigh->nud_state & (NUD_CONNECTED | NUD_DELAY | NUD_PROBE)))
+       if (!(READ_ONCE(neigh->nud_state) & (NUD_CONNECTED | NUD_DELAY | NUD_PROBE)))
                return __neigh_event_send(neigh, skb, immediate_ok);
        return 0;
 }

diff --git a/net/bridge/br_arp_nd_proxy.c b/net/bridge/br_arp_nd_proxy.c
index e5e48c6e35d786c63ab14b7df59caad3ef799633..b45c00c01dea195cf41beb069fa20a883926c9d3 100644 (file)
--- a/net/bridge/br_arp_nd_proxy.c
+++ b/net/bridge/br_arp_nd_proxy.c
@@ -192,7 +192,7 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br,
        if (n) {
                struct net_bridge_fdb_entry *f;
 
-               if (!(n->nud_state & NUD_VALID)) {
+               if (!(READ_ONCE(n->nud_state) & NUD_VALID)) {
                        neigh_release(n);
                        return;
                }
@@ -452,7 +452,7 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br,
        if (n) {
                struct net_bridge_fdb_entry *f;
 
-               if (!(n->nud_state & NUD_VALID)) {
+               if (!(READ_ONCE(n->nud_state) & NUD_VALID)) {
                        neigh_release(n);
                        return;
                }

diff --git a/net/bridge/br_netfilter_hooks.c b/net/bridge/br_netfilter_hooks.c
index 812bd7e1750b69a000d004984a9da7f987e52b6e..442198cb83909ee240a50b9b523fb8921c68457a 100644 (file)
--- a/net/bridge/br_netfilter_hooks.c
+++ b/net/bridge/br_netfilter_hooks.c
@@ -277,7 +277,8 @@ int br_nf_pre_routing_finish_bridge(struct net *net, struct sock *sk, struct sk_
                struct nf_bridge_info *nf_bridge = nf_bridge_info_get(skb);
                int ret;
 
-               if ((neigh->nud_state & NUD_CONNECTED) && neigh->hh.hh_len) {
+               if ((READ_ONCE(neigh->nud_state) & NUD_CONNECTED) &&
+                   READ_ONCE(neigh->hh.hh_len)) {
                        neigh_hh_bridge(&neigh->hh, skb);
                        skb->dev = nf_bridge->physindev;
                        ret = br_handle_frame_finish(net, sk, skb);

diff --git a/net/core/filter.c b/net/core/filter.c
index 6ef62d84dcac52031d0ae498a47c1723294fc1cf..d32c6bd4d579a8af88e7d0629a637bb796901eea 100644 (file)
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -5802,7 +5802,7 @@ static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params,
        else
                neigh = __ipv6_neigh_lookup_noref_stub(dev, params->ipv6_dst);
 
-       if (!neigh || !(neigh->nud_state & NUD_VALID))
+       if (!neigh || !(READ_ONCE(neigh->nud_state) & NUD_VALID))
                return BPF_FIB_LKUP_RET_NO_NEIGH;
        memcpy(params->dmac, neigh->ha, ETH_ALEN);
        memcpy(params->smac, dev->dev_addr, ETH_ALEN);
@@ -5923,7 +5923,7 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params,
         * not needed here.
         */
        neigh = __ipv6_neigh_lookup_noref_stub(dev, dst);
-       if (!neigh || !(neigh->nud_state & NUD_VALID))
+       if (!neigh || !(READ_ONCE(neigh->nud_state) & NUD_VALID))
                return BPF_FIB_LKUP_RET_NO_NEIGH;
        memcpy(params->dmac, neigh->ha, ETH_ALEN);
        memcpy(params->smac, dev->dev_addr, ETH_ALEN);

diff --git a/net/core/neighbour.c b/net/core/neighbour.c
index 8e726eb0548f6c26a2f9b2ba1a0b091da431f3d8..51393079487ae3bf501089b9faf236498a1078ae 100644 (file)
--- a/net/core/neighbour.c
+++ b/net/core/neighbour.c
@@ -1095,13 +1095,13 @@ static void neigh_timer_handler(struct timer_list *t)
                                          neigh->used +
                                          NEIGH_VAR(neigh->parms, DELAY_PROBE_TIME))) {
                        neigh_dbg(2, "neigh %p is delayed\n", neigh);
-                       neigh->nud_state = NUD_DELAY;
+                       WRITE_ONCE(neigh->nud_state, NUD_DELAY);
                        neigh->updated = jiffies;
                        neigh_suspect(neigh);
                        next = now + NEIGH_VAR(neigh->parms, DELAY_PROBE_TIME);
                } else {
                        neigh_dbg(2, "neigh %p is suspected\n", neigh);
-                       neigh->nud_state = NUD_STALE;
+                       WRITE_ONCE(neigh->nud_state, NUD_STALE);
                        neigh->updated = jiffies;
                        neigh_suspect(neigh);
                        notify = 1;
@@ -1111,14 +1111,14 @@ static void neigh_timer_handler(struct timer_list *t)
                                   neigh->confirmed +
                                   NEIGH_VAR(neigh->parms, DELAY_PROBE_TIME))) {
                        neigh_dbg(2, "neigh %p is now reachable\n", neigh);
-                       neigh->nud_state = NUD_REACHABLE;
+                       WRITE_ONCE(neigh->nud_state, NUD_REACHABLE);
                        neigh->updated = jiffies;
                        neigh_connect(neigh);
                        notify = 1;
                        next = neigh->confirmed + neigh->parms->reachable_time;
                } else {
                        neigh_dbg(2, "neigh %p is probed\n", neigh);
-                       neigh->nud_state = NUD_PROBE;
+                       WRITE_ONCE(neigh->nud_state, NUD_PROBE);
                        neigh->updated = jiffies;
                        atomic_set(&neigh->probes, 0);
                        notify = 1;
@@ -1132,7 +1132,7 @@ static void neigh_timer_handler(struct timer_list *t)
 
        if ((neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) &&
            atomic_read(&neigh->probes) >= neigh_max_probes(neigh)) {
-               neigh->nud_state = NUD_FAILED;
+               WRITE_ONCE(neigh->nud_state, NUD_FAILED);
                notify = 1;
                neigh_invalidate(neigh);
                goto out;
@@ -1181,7 +1181,7 @@ int __neigh_event_send(struct neighbour *neigh, struct sk_buff *skb,
                        atomic_set(&neigh->probes,
                                   NEIGH_VAR(neigh->parms, UCAST_PROBES));
                        neigh_del_timer(neigh);
-                       neigh->nud_state = NUD_INCOMPLETE;
+                       WRITE_ONCE(neigh->nud_state, NUD_INCOMPLETE);
                        neigh->updated = now;
                        if (!immediate_ok) {
                                next = now + 1;
@@ -1193,7 +1193,7 @@ int __neigh_event_send(struct neighbour *neigh, struct sk_buff *skb,
                        }
                        neigh_add_timer(neigh, next);
                } else {
-                       neigh->nud_state = NUD_FAILED;
+                       WRITE_ONCE(neigh->nud_state, NUD_FAILED);
                        neigh->updated = jiffies;
                        write_unlock_bh(&neigh->lock);
 
@@ -1203,7 +1203,7 @@ int __neigh_event_send(struct neighbour *neigh, struct sk_buff *skb,
        } else if (neigh->nud_state & NUD_STALE) {
                neigh_dbg(2, "neigh %p is delayed\n", neigh);
                neigh_del_timer(neigh);
-               neigh->nud_state = NUD_DELAY;
+               WRITE_ONCE(neigh->nud_state, NUD_DELAY);
                neigh->updated = jiffies;
                neigh_add_timer(neigh, jiffies +
                                NEIGH_VAR(neigh->parms, DELAY_PROBE_TIME));
@@ -1315,7 +1315,7 @@ static int __neigh_update(struct neighbour *neigh, const u8 *lladdr,
        neigh_update_flags(neigh, flags, &notify, &gc_update, &managed_update);
        if (flags & (NEIGH_UPDATE_F_USE | NEIGH_UPDATE_F_MANAGED)) {
                new = old & ~NUD_PERMANENT;
-               neigh->nud_state = new;
+               WRITE_ONCE(neigh->nud_state, new);
                err = 0;
                goto out;
        }
@@ -1324,7 +1324,7 @@ static int __neigh_update(struct neighbour *neigh, const u8 *lladdr,
                neigh_del_timer(neigh);
                if (old & NUD_CONNECTED)
                        neigh_suspect(neigh);
-               neigh->nud_state = new;
+               WRITE_ONCE(neigh->nud_state, new);
                err = 0;
                notify = old & NUD_VALID;
                if ((old & (NUD_INCOMPLETE | NUD_PROBE)) &&
@@ -1403,7 +1403,7 @@ static int __neigh_update(struct neighbour *neigh, const u8 *lladdr,
                                                ((new & NUD_REACHABLE) ?
                                                 neigh->parms->reachable_time :
                                                 0)));
-               neigh->nud_state = new;
+               WRITE_ONCE(neigh->nud_state, new);
                notify = 1;
        }
 
@@ -1490,7 +1490,7 @@ void __neigh_set_probe_once(struct neighbour *neigh)
        neigh->updated = jiffies;
        if (!(neigh->nud_state & NUD_FAILED))
                return;
-       neigh->nud_state = NUD_INCOMPLETE;
+       WRITE_ONCE(neigh->nud_state, NUD_INCOMPLETE);
        atomic_set(&neigh->probes, neigh_max_probes(neigh));
        neigh_add_timer(neigh,
                        jiffies + max(NEIGH_VAR(neigh->parms, RETRANS_TIME),
@@ -3190,7 +3190,7 @@ static struct neighbour *neigh_get_first(struct seq_file *seq)
                        }
                        if (!(state->flags & NEIGH_SEQ_SKIP_NOARP))
                                break;
-                       if (n->nud_state & ~NUD_NOARP)
+                       if (READ_ONCE(n->nud_state) & ~NUD_NOARP)
                                break;
 next:
                        n = rcu_dereference_bh(n->next);
@@ -3232,7 +3232,7 @@ static struct neighbour *neigh_get_next(struct seq_file *seq,
                        if (!(state->flags & NEIGH_SEQ_SKIP_NOARP))
                                break;
 
-                       if (n->nud_state & ~NUD_NOARP)
+                       if (READ_ONCE(n->nud_state) & ~NUD_NOARP)
                                break;
 next:
                        n = rcu_dereference_bh(n->next);

diff --git a/net/ipv4/arp.c b/net/ipv4/arp.c
index 4f7237661afb9b2c1a11a1740eae823443ebdea4..9456f5bb35e5d9e97d6c05be21561b435e2b704a 100644 (file)
--- a/net/ipv4/arp.c
+++ b/net/ipv4/arp.c
@@ -375,7 +375,7 @@ static void arp_solicit(struct neighbour *neigh, struct sk_buff *skb)
 
        probes -= NEIGH_VAR(neigh->parms, UCAST_PROBES);
        if (probes < 0) {
-               if (!(neigh->nud_state & NUD_VALID))
+               if (!(READ_ONCE(neigh->nud_state) & NUD_VALID))
                        pr_debug("trying to ucast probe in NUD_INVALID\n");
                neigh_ha_snapshot(dst_ha, neigh, dev);
                dst_hw = dst_ha;
@@ -1123,7 +1123,7 @@ static int arp_req_get(struct arpreq *r, struct net_device *dev)
 
        neigh = neigh_lookup(&arp_tbl, &ip, dev);
        if (neigh) {
-               if (!(neigh->nud_state & NUD_NOARP)) {
+               if (!(READ_ONCE(neigh->nud_state) & NUD_NOARP)) {
                        read_lock_bh(&neigh->lock);
                        memcpy(r->arp_ha.sa_data, neigh->ha, dev->addr_len);
                        r->arp_flags = arp_state_to_flags(neigh);
@@ -1144,12 +1144,12 @@ int arp_invalidate(struct net_device *dev, __be32 ip, bool force)
        struct neigh_table *tbl = &arp_tbl;
 
        if (neigh) {
-               if ((neigh->nud_state & NUD_VALID) && !force) {
+               if ((READ_ONCE(neigh->nud_state) & NUD_VALID) && !force) {
                        neigh_release(neigh);
                        return 0;
                }
 
-               if (neigh->nud_state & ~NUD_NOARP)
+               if (READ_ONCE(neigh->nud_state) & ~NUD_NOARP)
                        err = neigh_update(neigh, NULL, NUD_FAILED,
                                           NEIGH_UPDATE_F_OVERRIDE|
                                           NEIGH_UPDATE_F_ADMIN, 0);

diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c
index 3b6e6bc80dc1cfd66d66f47b86f14e1767d7f79c..158ad3c2662f50827724b2f348ed15d922392508 100644 (file)
--- a/net/ipv4/fib_semantics.c
+++ b/net/ipv4/fib_semantics.c
@@ -564,7 +564,7 @@ static int fib_detect_death(struct fib_info *fi, int order,
                n = NULL;
 
        if (n) {
-               state = n->nud_state;
+               state = READ_ONCE(n->nud_state);
                neigh_release(n);
        } else {
                return 0;
@@ -2205,7 +2205,7 @@ static bool fib_good_nh(const struct fib_nh *nh)
                else
                        n = NULL;
                if (n)
-                       state = n->nud_state;
+                       state = READ_ONCE(n->nud_state);
 
                rcu_read_unlock_bh();
        }

diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c
index 9cc2879024541d432cac268472102cd19a6d7602..d699a41c9d9557e691cfd986c03f97bce21986d2 100644 (file)
--- a/net/ipv4/nexthop.c
+++ b/net/ipv4/nexthop.c
@@ -1128,7 +1128,7 @@ static bool ipv6_good_nh(const struct fib6_nh *nh)
 
        n = __ipv6_neigh_lookup_noref_stub(nh->fib_nh_dev, &nh->fib_nh_gw6);
        if (n)
-               state = n->nud_state;
+               state = READ_ONCE(n->nud_state);
 
        rcu_read_unlock_bh();
 
@@ -1145,7 +1145,7 @@ static bool ipv4_good_nh(const struct fib_nh *nh)
        n = __ipv4_neigh_lookup_noref(nh->fib_nh_dev,
                                      (__force u32)nh->fib_nh_gw4);
        if (n)
-               state = n->nud_state;
+               state = READ_ONCE(n->nud_state);
 
        rcu_read_unlock_bh();
 

diff --git a/net/ipv4/route.c b/net/ipv4/route.c
index 84a0a71a6f4e7c63de6ea86936da67079613c1cb..8d838b00469002732f52b305c27b6432882a3ee1 100644 (file)
--- a/net/ipv4/route.c
+++ b/net/ipv4/route.c
@@ -784,7 +784,7 @@ static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flow
        if (!n)
                n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
        if (!IS_ERR(n)) {
-               if (!(n->nud_state & NUD_VALID)) {
+               if (!(READ_ONCE(n->nud_state) & NUD_VALID)) {
                        neigh_event_send(n, NULL);
                } else {
                        if (fib_lookup(net, fl4, &res, 0) == 0) {

diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
index 34192f7a166fbc42cae749e56ba77ec4298ee584..5bf15a530fe73c6f74088b478c652d2fa2dc37eb 100644 (file)
--- a/net/ipv6/ip6_output.c
+++ b/net/ipv6/ip6_output.c
@@ -1153,7 +1153,7 @@ static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk,
        rcu_read_lock_bh();
        n = __ipv6_neigh_lookup_noref(rt->dst.dev,
                                      rt6_nexthop(rt, &fl6->daddr));
-       err = n && !(n->nud_state & NUD_VALID) ? -EINVAL : 0;
+       err = n && !(READ_ONCE(n->nud_state) & NUD_VALID) ? -EINVAL : 0;
        rcu_read_unlock_bh();
 
        if (err) {

diff --git a/net/ipv6/ndisc.c b/net/ipv6/ndisc.c
index 6cb2d6a536a84ec3ef89bf1143f9aceaaf7ae9c2..8c5a99fe68030293e9d3f6de7542318617587e82 100644 (file)
--- a/net/ipv6/ndisc.c
+++ b/net/ipv6/ndisc.c
@@ -746,7 +746,7 @@ static void ndisc_solicit(struct neighbour *neigh, struct sk_buff *skb)
                saddr = &ipv6_hdr(skb)->saddr;
        probes -= NEIGH_VAR(neigh->parms, UCAST_PROBES);
        if (probes < 0) {
-               if (!(neigh->nud_state & NUD_VALID)) {
+               if (!(READ_ONCE(neigh->nud_state) & NUD_VALID)) {
                        ND_PRINTK(1, dbg,
                                  "%s: trying to ucast probe in NUD_INVALID: %pI6\n",
                                  __func__, target);
@@ -1092,7 +1092,7 @@ static void ndisc_recv_na(struct sk_buff *skb)
                u8 old_flags = neigh->flags;
                struct net *net = dev_net(dev);
 
-               if (neigh->nud_state & NUD_FAILED)
+               if (READ_ONCE(neigh->nud_state) & NUD_FAILED)
                        goto out;
 
                /*

diff --git a/net/ipv6/route.c b/net/ipv6/route.c
index 93957b20fccceed2e1e56432fe76bd9a6dce780f..4f103e7c4ea25f3162bb2cc8f62497aa863f5878 100644 (file)
--- a/net/ipv6/route.c
+++ b/net/ipv6/route.c
@@ -641,7 +641,7 @@ static void rt6_probe(struct fib6_nh *fib6_nh)
        idev = __in6_dev_get(dev);
        neigh = __ipv6_neigh_lookup_noref(dev, nh_gw);
        if (neigh) {
-               if (neigh->nud_state & NUD_VALID)
+               if (READ_ONCE(neigh->nud_state) & NUD_VALID)
                        goto out;
 
                write_lock(&neigh->lock);