i40e: read the XDP program once per NAPI

Similar to how it's done in the ice driver since 'd241f103f9c0 ("ice:
propagate xdp_ring onto rx_ring")', read the XDP program once per NAPI
instead of once per descriptor cleaned. I measured an improvement in
throughput of 2% for the AF_XDP xdpsock l2fwd benchmark for zero copy mode
and 1% for copy mode.

Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Link: https://lore.kernel.org/r/20220623100852.7867-1-ciara.loftus@intel.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
index dadef56e5f9bd04161e3b3762d42dc1b94767f11..a327189deda0978d5c9752954e769dc9068a4679 100644 (file)
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
@@ -2289,16 +2289,14 @@ int i40e_xmit_xdp_tx_ring(struct xdp_buff *xdp, struct i40e_ring *xdp_ring)
  * i40e_run_xdp - run an XDP program
  * @rx_ring: Rx ring being processed
  * @xdp: XDP buffer containing the frame
+ * @xdp_prog: XDP program to run
  **/
-static int i40e_run_xdp(struct i40e_ring *rx_ring, struct xdp_buff *xdp)
+static int i40e_run_xdp(struct i40e_ring *rx_ring, struct xdp_buff *xdp, struct bpf_prog *xdp_prog)
 {
        int err, result = I40E_XDP_PASS;
        struct i40e_ring *xdp_ring;
-       struct bpf_prog *xdp_prog;
        u32 act;
 
-       xdp_prog = READ_ONCE(rx_ring->xdp_prog);
-
        if (!xdp_prog)
                goto xdp_out;
 
@@ -2443,6 +2441,7 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
        unsigned int offset = rx_ring->rx_offset;
        struct sk_buff *skb = rx_ring->skb;
        unsigned int xdp_xmit = 0;
+       struct bpf_prog *xdp_prog;
        bool failure = false;
        struct xdp_buff xdp;
        int xdp_res = 0;
@@ -2452,6 +2451,8 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
 #endif
        xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq);
 
+       xdp_prog = READ_ONCE(rx_ring->xdp_prog);
+
        while (likely(total_rx_packets < (unsigned int)budget)) {
                struct i40e_rx_buffer *rx_buffer;
                union i40e_rx_desc *rx_desc;
@@ -2512,7 +2513,7 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
                        /* At larger PAGE_SIZE, frame_sz depend on len size */
                        xdp.frame_sz = i40e_rx_frame_truesize(rx_ring, size);
 #endif
-                       xdp_res = i40e_run_xdp(rx_ring, &xdp);
+                       xdp_res = i40e_run_xdp(rx_ring, &xdp, xdp_prog);
                }
 
                if (xdp_res) {

diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
index af3e7e6afc8528e86c45ccce10f71744db68293a..6d4009e0cbd625f52c6f2da02f1457ad516a5759 100644 (file)
--- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
@@ -143,20 +143,17 @@ int i40e_xsk_pool_setup(struct i40e_vsi *vsi, struct xsk_buff_pool *pool,
  * i40e_run_xdp_zc - Executes an XDP program on an xdp_buff
  * @rx_ring: Rx ring
  * @xdp: xdp_buff used as input to the XDP program
+ * @xdp_prog: XDP program to run
  *
  * Returns any of I40E_XDP_{PASS, CONSUMED, TX, REDIR}
  **/
-static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp)
+static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp,
+                          struct bpf_prog *xdp_prog)
 {
        int err, result = I40E_XDP_PASS;
        struct i40e_ring *xdp_ring;
-       struct bpf_prog *xdp_prog;
        u32 act;
 
-       /* NB! xdp_prog will always be !NULL, due to the fact that
-        * this path is enabled by setting an XDP program.
-        */
-       xdp_prog = READ_ONCE(rx_ring->xdp_prog);
        act = bpf_prog_run_xdp(xdp_prog, xdp);
 
        if (likely(act == XDP_REDIRECT)) {
@@ -339,9 +336,15 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
        u16 next_to_clean = rx_ring->next_to_clean;
        u16 count_mask = rx_ring->count - 1;
        unsigned int xdp_res, xdp_xmit = 0;
+       struct bpf_prog *xdp_prog;
        bool failure = false;
        u16 cleaned_count;
 
+       /* NB! xdp_prog will always be !NULL, due to the fact that
+        * this path is enabled by setting an XDP program.
+        */
+       xdp_prog = READ_ONCE(rx_ring->xdp_prog);
+
        while (likely(total_rx_packets < (unsigned int)budget)) {
                union i40e_rx_desc *rx_desc;
                unsigned int rx_packets;
@@ -378,7 +381,7 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
                xsk_buff_set_size(bi, size);
                xsk_buff_dma_sync_for_cpu(bi, rx_ring->xsk_pool);
 
-               xdp_res = i40e_run_xdp_zc(rx_ring, bi);
+               xdp_res = i40e_run_xdp_zc(rx_ring, bi, xdp_prog);
                i40e_handle_xdp_result_zc(rx_ring, bi, rx_desc, &rx_packets,
                                          &rx_bytes, size, xdp_res, &failure);
                if (failure)