gve: Move the irq db indexes out of the ntfy block struct

Giving the device access to other kernel structs is not ideal.
Move the indexes into their own array and just keep pointers to
them in the ntfy block struct.

Signed-off-by: Catherine Sullivan <csully@google.com>
Signed-off-by: David Awogbemila <awogbemila@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index b719f72281c44b9798c67d7b936217f44143fe73..b6bd8f679127ed27ed228c796f85899cc8077547 100644 (file)
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -441,13 +441,13 @@ struct gve_tx_ring {
  * associated with that irq.
  */
 struct gve_notify_block {
-       __be32 irq_db_index; /* idx into Bar2 - set by device, must be 1st */
+       __be32 *irq_db_index; /* pointer to idx into Bar2 */
        char name[IFNAMSIZ + 16]; /* name registered with the kernel */
        struct napi_struct napi; /* kernel napi struct for this block */
        struct gve_priv *priv;
        struct gve_tx_ring *tx; /* tx rings on this block */
        struct gve_rx_ring *rx; /* rx rings on this block */
-} ____cacheline_aligned;
+};
 
 /* Tracks allowed and current queue settings */
 struct gve_queue_config {
@@ -466,6 +466,10 @@ struct gve_options_dqo_rda {
        u16 rx_buff_ring_entries; /* number of rx_buff descriptors */
 };
 
+struct gve_irq_db {
+       __be32 index;
+} ____cacheline_aligned;
+
 struct gve_ptype {
        u8 l3_type;  /* `gve_l3_type` in gve_adminq.h */
        u8 l4_type;  /* `gve_l4_type` in gve_adminq.h */
@@ -492,7 +496,8 @@ struct gve_priv {
        struct gve_rx_ring *rx; /* array of rx_cfg.num_queues */
        struct gve_queue_page_list *qpls; /* array of num qpls */
        struct gve_notify_block *ntfy_blocks; /* array of num_ntfy_blks */
-       dma_addr_t ntfy_block_bus;
+       struct gve_irq_db *irq_db_indices; /* array of num_ntfy_blks */
+       dma_addr_t irq_db_indices_bus;
        struct msix_entry *msix_vectors; /* array of num_ntfy_blks + 1 */
        char mgmt_msix_name[IFNAMSIZ + 16];
        u32 mgmt_msix_idx;
@@ -733,7 +738,7 @@ static inline void gve_clear_report_stats(struct gve_priv *priv)
 static inline __be32 __iomem *gve_irq_doorbell(struct gve_priv *priv,
                                               struct gve_notify_block *block)
 {
-       return &priv->db_bar2[be32_to_cpu(block->irq_db_index)];
+       return &priv->db_bar2[be32_to_cpu(*block->irq_db_index)];
 }
 
 /* Returns the index into ntfy_blocks of the given tx ring's block

diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index 326b56b49216a8c4e1d3868384622bfa81860f63..2ad7f57f7e5b52780493d647d384684a65169df0 100644 (file)
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -462,7 +462,7 @@ int gve_adminq_configure_device_resources(struct gve_priv *priv,
                .num_counters = cpu_to_be32(num_counters),
                .irq_db_addr = cpu_to_be64(db_array_bus_addr),
                .num_irq_dbs = cpu_to_be32(num_ntfy_blks),
-               .irq_db_stride = cpu_to_be32(sizeof(priv->ntfy_blocks[0])),
+               .irq_db_stride = cpu_to_be32(sizeof(*priv->irq_db_indices)),
                .ntfy_blk_msix_base_idx =
                                        cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX),
                .queue_format = priv->queue_format,

diff --git a/drivers/net/ethernet/google/gve/gve_dqo.h b/drivers/net/ethernet/google/gve/gve_dqo.h
index 836042364124e123537ccf1e5a880fe493c4a2d3..b2e2fb01569316dea6d5b0d69e4c5ad42e8cab2c 100644 (file)
--- a/drivers/net/ethernet/google/gve/gve_dqo.h
+++ b/drivers/net/ethernet/google/gve/gve_dqo.h
@@ -73,7 +73,7 @@ static inline void
 gve_write_irq_doorbell_dqo(const struct gve_priv *priv,
                           const struct gve_notify_block *block, u32 val)
 {
-       u32 index = be32_to_cpu(block->irq_db_index);
+       u32 index = be32_to_cpu(*block->irq_db_index);
 
        iowrite32(val, &priv->db_bar2[index]);
 }

diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 59b66f679e46e896f122da9cee171ff301968735..348b4cfc4a12c524a4357f376e1be92287ef06c3 100644 (file)
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -334,15 +334,23 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
                dev_err(&priv->pdev->dev, "Did not receive management vector.\n");
                goto abort_with_msix_enabled;
        }
-       priv->ntfy_blocks =
+       priv->irq_db_indices =
                dma_alloc_coherent(&priv->pdev->dev,
                                   priv->num_ntfy_blks *
-                                  sizeof(*priv->ntfy_blocks),
-                                  &priv->ntfy_block_bus, GFP_KERNEL);
-       if (!priv->ntfy_blocks) {
+                                  sizeof(*priv->irq_db_indices),
+                                  &priv->irq_db_indices_bus, GFP_KERNEL);
+       if (!priv->irq_db_indices) {
                err = -ENOMEM;
                goto abort_with_mgmt_vector;
        }
+
+       priv->ntfy_blocks = kvzalloc(priv->num_ntfy_blks *
+                                    sizeof(*priv->ntfy_blocks), GFP_KERNEL);
+       if (!priv->ntfy_blocks) {
+               err = -ENOMEM;
+               goto abort_with_irq_db_indices;
+       }
+
        /* Setup the other blocks - the first n-1 vectors */
        for (i = 0; i < priv->num_ntfy_blks; i++) {
                struct gve_notify_block *block = &priv->ntfy_blocks[i];
@@ -361,6 +369,7 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
                }
                irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
                                      get_cpu_mask(i % active_cpus));
+               block->irq_db_index = &priv->irq_db_indices[i].index;
        }
        return 0;
 abort_with_some_ntfy_blocks:
@@ -372,10 +381,13 @@ abort_with_some_ntfy_blocks:
                                      NULL);
                free_irq(priv->msix_vectors[msix_idx].vector, block);
        }
-       dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
-                         sizeof(*priv->ntfy_blocks),
-                         priv->ntfy_blocks, priv->ntfy_block_bus);
+       kvfree(priv->ntfy_blocks);
        priv->ntfy_blocks = NULL;
+abort_with_irq_db_indices:
+       dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
+                         sizeof(*priv->irq_db_indices),
+                         priv->irq_db_indices, priv->irq_db_indices_bus);
+       priv->irq_db_indices = NULL;
 abort_with_mgmt_vector:
        free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
 abort_with_msix_enabled:
@@ -403,10 +415,12 @@ static void gve_free_notify_blocks(struct gve_priv *priv)
                free_irq(priv->msix_vectors[msix_idx].vector, block);
        }
        free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
-       dma_free_coherent(&priv->pdev->dev,
-                         priv->num_ntfy_blks * sizeof(*priv->ntfy_blocks),
-                         priv->ntfy_blocks, priv->ntfy_block_bus);
+       kvfree(priv->ntfy_blocks);
        priv->ntfy_blocks = NULL;
+       dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
+                         sizeof(*priv->irq_db_indices),
+                         priv->irq_db_indices, priv->irq_db_indices_bus);
+       priv->irq_db_indices = NULL;
        pci_disable_msix(priv->pdev);
        kvfree(priv->msix_vectors);
        priv->msix_vectors = NULL;
@@ -428,7 +442,7 @@ static int gve_setup_device_resources(struct gve_priv *priv)
        err = gve_adminq_configure_device_resources(priv,
                                                    priv->counter_array_bus,
                                                    priv->num_event_counters,
-                                                   priv->ntfy_block_bus,
+                                                   priv->irq_db_indices_bus,
                                                    priv->num_ntfy_blks);
        if (unlikely(err)) {
                dev_err(&priv->pdev->dev,