net/mlx5e: Disable softirq in mlx5e_activate_rq to avoid race condition

When the driver activates the channels, it assumes NAPI isn't running
yet. mlx5e_activate_rq posts a NOP WQE to ICOSQ to trigger a hardware
interrupt and start NAPI, which will run mlx5e_alloc_rx_mpwqe and post
UMR WQEs to ICOSQ to be able to receive packets with striding RQ.

Unfortunately, a race condition is possible if NAPI is triggered by
something else (for example, TX) at a bad timing, before
mlx5e_activate_rq finishes. In this case, mlx5e_alloc_rx_mpwqe may post
UMR WQEs to ICOSQ, and with the bad timing, the wqe_info of the first
UMR may be overwritten by the wqe_info of the NOP posted by
mlx5e_activate_rq.

The consequence is that icosq->db.wqe_info[0].num_wqebbs will be changed
from MLX5E_UMR_WQEBBS to 1, disrupting the integrity of the array-based
linked list in wqe_info[]. mlx5e_poll_ico_cq will hang in an infinite
loop after processing wqe_info[0], because after the corruption, the
next item to be processed will be wqe_info[1], which is filled with
zeros, and `sqcc += wi->num_wqebbs` will never move further.

This commit fixes this race condition by using async_icosq to post the
NOP and trigger the interrupt. async_icosq is always protected with a
spinlock, eliminating the race condition.

Fixes: 3019ec658783 ("net/mlx5e: Add fragmented memory support for RX multi packet WQE")
Signed-off-by: Maxim Mikityanskiy <maximmi@nvidia.com>
Reported-by: Karsten Nielsen <karsten@foo-bar.dk>
Reviewed-by: Tariq Toukan <tariqt@nvidia.com>
Reviewed-by: Gal Pressman <gal@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 65d3c4865abf71a7cc84a33e2951fd2d6f75f6ec..b6c15efe92ad496883095b45535f7ab04fdb8919 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -764,6 +764,7 @@ struct mlx5e_rq {
        u8                     wq_type;
        u32                    rqn;
        struct mlx5_core_dev  *mdev;
+       struct mlx5e_channel  *channel;
        u32  umr_mkey;
        struct mlx5e_dma_info  wqe_overflow;
 
@@ -1076,6 +1077,9 @@ void mlx5e_close_cq(struct mlx5e_cq *cq);
 int mlx5e_open_locked(struct net_device *netdev);
 int mlx5e_close_locked(struct net_device *netdev);
 
+void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c);
+void mlx5e_trigger_napi_sched(struct napi_struct *napi);
+
 int mlx5e_open_channels(struct mlx5e_priv *priv,
                        struct mlx5e_channels *chs);
 void mlx5e_close_channels(struct mlx5e_channels *chs);

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
index 335b20b6383b6447360fef2a937f9d0e9957cd5b..047f88f0920387e2b65cd1aca5ffaeb0c4540199 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
@@ -736,6 +736,7 @@ void mlx5e_ptp_activate_channel(struct mlx5e_ptp *c)
        if (test_bit(MLX5E_PTP_STATE_RX, c->state)) {
                mlx5e_ptp_rx_set_fs(c->priv);
                mlx5e_activate_rq(&c->rq);
+               mlx5e_trigger_napi_sched(&c->napi);
        }
 }
 

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c
index 2684e9da9f41251f2e3fb1f008cb6e89fab1a487..fc366e66d0b0f98ed431845522fa82f6b3b01389 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c
@@ -123,6 +123,8 @@ static int mlx5e_rx_reporter_err_icosq_cqe_recover(void *ctx)
                xskrq->stats->recover++;
        }
 
+       mlx5e_trigger_napi_icosq(icosq->channel);
+
        mutex_unlock(&icosq->channel->icosq_recovery_lock);
 
        return 0;
@@ -166,6 +168,10 @@ static int mlx5e_rx_reporter_err_rq_cqe_recover(void *ctx)
        clear_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state);
        mlx5e_activate_rq(rq);
        rq->stats->recover++;
+       if (rq->channel)
+               mlx5e_trigger_napi_icosq(rq->channel);
+       else
+               mlx5e_trigger_napi_sched(rq->cq.napi);
        return 0;
 out:
        clear_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state);

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c
index 857840ab1e91885d7a22810fc9b86505663f0299..11f2a7fb72a9e620e6d37da3037baddcee5a7552 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c
@@ -179,6 +179,7 @@ static void mlx5e_activate_trap(struct mlx5e_trap *trap)
 {
        napi_enable(&trap->napi);
        mlx5e_activate_rq(&trap->rq);
+       mlx5e_trigger_napi_sched(&trap->napi);
 }
 
 void mlx5e_deactivate_trap(struct mlx5e_priv *priv)

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
index 279cd8f4e79f7236e728691dd2de5c4d3f538ab0..2c520394aa1d646c4bb3f30ee46fa98bf4bd5027 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
@@ -117,6 +117,7 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
                goto err_remove_pool;
 
        mlx5e_activate_xsk(c);
+       mlx5e_trigger_napi_icosq(c);
 
        /* Don't wait for WQEs, because the newer xdpsock sample doesn't provide
         * any Fill Ring entries at the setup stage.

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
index 3ad7f1301fa8d06afbe0ec72494c295a87020d90..98ed9ef3a6bdd02277eabdccade2ba380ca09b4a 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
@@ -64,6 +64,7 @@ static int mlx5e_init_xsk_rq(struct mlx5e_channel *c,
        rq->clock        = &mdev->clock;
        rq->icosq        = &c->icosq;
        rq->ix           = c->ix;
+       rq->channel      = c;
        rq->mdev         = mdev;
        rq->hw_mtu       = MLX5E_SW2HW_MTU(params, params->sw_mtu);
        rq->xdpsq        = &c->rq_xdpsq;
@@ -179,10 +180,6 @@ void mlx5e_activate_xsk(struct mlx5e_channel *c)
        mlx5e_reporter_icosq_resume_recovery(c);
 
        /* TX queue is created active. */
-
-       spin_lock_bh(&c->async_icosq_lock);
-       mlx5e_trigger_irq(&c->async_icosq);
-       spin_unlock_bh(&c->async_icosq_lock);
 }
 
 void mlx5e_deactivate_xsk(struct mlx5e_channel *c)

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 05c015515ccedf493cb60e35b6a8d0b9400d76cb..930a5402c817c1c25bf4d4bbd2f5317758ad5f78 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -475,6 +475,7 @@ static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *param
        rq->clock        = &mdev->clock;
        rq->icosq        = &c->icosq;
        rq->ix           = c->ix;
+       rq->channel      = c;
        rq->mdev         = mdev;
        rq->hw_mtu       = MLX5E_SW2HW_MTU(params, params->sw_mtu);
        rq->xdpsq        = &c->rq_xdpsq;
@@ -1066,13 +1067,6 @@ err_free_rq:
 void mlx5e_activate_rq(struct mlx5e_rq *rq)
 {
        set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
-       if (rq->icosq) {
-               mlx5e_trigger_irq(rq->icosq);
-       } else {
-               local_bh_disable();
-               napi_schedule(rq->cq.napi);
-               local_bh_enable();
-       }
 }
 
 void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
@@ -2227,6 +2221,20 @@ static int mlx5e_channel_stats_alloc(struct mlx5e_priv *priv, int ix, int cpu)
        return 0;
 }
 
+void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c)
+{
+       spin_lock_bh(&c->async_icosq_lock);
+       mlx5e_trigger_irq(&c->async_icosq);
+       spin_unlock_bh(&c->async_icosq_lock);
+}
+
+void mlx5e_trigger_napi_sched(struct napi_struct *napi)
+{
+       local_bh_disable();
+       napi_schedule(napi);
+       local_bh_enable();
+}
+
 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
                              struct mlx5e_params *params,
                              struct mlx5e_channel_param *cparam,
@@ -2308,6 +2316,8 @@ static void mlx5e_activate_channel(struct mlx5e_channel *c)
 
        if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
                mlx5e_activate_xsk(c);
+
+       mlx5e_trigger_napi_icosq(c);
 }
 
 static void mlx5e_deactivate_channel(struct mlx5e_channel *c)