From d6dcf91499125fa8c0c6c7b9a6e17ec5bd22e5ef Mon Sep 17 00:00:00 2001 From: Frank Li Date: Tue, 24 May 2022 10:21:59 -0500 Subject: [PATCH] PCI: endpoint: Enable DMA tests for endpoints with DMA capabilities Some PCI Endpoint controllers integrate an eDMA (embedded DMA). eDMA can bypass the outbound memory address translation unit to access all RC memory space. Add eDMA support for pci-epf-test. Depending on HW availability, the EPF test can use either eDMA or general system DMA controllers to perform DMA. The test tries to use eDMA first and falls back to general system DMA controllers if there's no eDMA Separate dma_chan to dma_chan_tx and dma_chan_rx. Search for an eDMA channel first, then search for a memory-to-memory DMA channel. If general memory to memory channels are used, dma_chan_rx = dma_chan_tx. Add dma_addr_t dma_remote in pci_epf_test_data_transfer() because eDMA uses remote RC physical address directly. Add enum dma_transfer_direction dir in pci_epf_test_data_transfer() because eDMA chooses the correct RX/TX channel by dir. The overall steps are: 1. Execute dma_request_channel() and filter function to find correct eDMA RX and TX Channel. If a channel does not exist, fallback to try to allocate general memory to memory DMA channel. 2. Execute dmaengine_slave_config() to configure remote side physical address. 3. Execute dmaengine_prep_slave_single() to create transfer descriptor. 4. Execute tx_submit(). 5. Execute dma_async_issue_pending() [bhelgaas: squash in fix from Dan Carpenter : https://lore.kernel.org/r/Ys2GSTnZhuLzzQG5@kili, also previously posted by Peng Wu : https://lore.kernel.org/all/CANXvt5rK98-cEMgpzopY9POOK8a5=VDib8uKPLgJakOG=hRfwQ@mail.gmail.com/] Link: https://lore.kernel.org/r/20220524152159.2370739-9-Frank.Li@nxp.com Signed-off-by: Frank Li Signed-off-by: Bjorn Helgaas Acked-by: Manivannan Sadhasivam Acked-by: Kishon Vijay Abraham I Acked-By: Vinod Koul --- drivers/pci/endpoint/functions/pci-epf-test.c | 116 ++++++++++++++++-- 1 file changed, 106 insertions(+), 10 deletions(-) diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index 5b833f00e9800..8586405d870ca 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -52,9 +52,11 @@ struct pci_epf_test { enum pci_barno test_reg_bar; size_t msix_table_offset; struct delayed_work cmd_handler; - struct dma_chan *dma_chan; + struct dma_chan *dma_chan_tx; + struct dma_chan *dma_chan_rx; struct completion transfer_complete; bool dma_supported; + bool dma_private; const struct pci_epc_features *epc_features; }; @@ -96,6 +98,8 @@ static void pci_epf_test_dma_callback(void *param) * @dma_src: The source address of the data transfer. It can be a physical * address given by pci_epc_mem_alloc_addr or DMA mapping APIs. * @len: The size of the data transfer + * @dma_remote: remote RC physical address + * @dir: DMA transfer direction * * Function that uses dmaengine API to transfer data between PCIe EP and remote * PCIe RC. The source and destination address can be a physical address given @@ -105,12 +109,16 @@ static void pci_epf_test_dma_callback(void *param) */ static int pci_epf_test_data_transfer(struct pci_epf_test *epf_test, dma_addr_t dma_dst, dma_addr_t dma_src, - size_t len) + size_t len, dma_addr_t dma_remote, + enum dma_transfer_direction dir) { + struct dma_chan *chan = (dir == DMA_DEV_TO_MEM) ? + epf_test->dma_chan_tx : epf_test->dma_chan_rx; + dma_addr_t dma_local = (dir == DMA_MEM_TO_DEV) ? dma_src : dma_dst; enum dma_ctrl_flags flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT; - struct dma_chan *chan = epf_test->dma_chan; struct pci_epf *epf = epf_test->epf; struct dma_async_tx_descriptor *tx; + struct dma_slave_config sconf = {}; struct device *dev = &epf->dev; dma_cookie_t cookie; int ret; @@ -120,7 +128,24 @@ static int pci_epf_test_data_transfer(struct pci_epf_test *epf_test, return -EINVAL; } - tx = dmaengine_prep_dma_memcpy(chan, dma_dst, dma_src, len, flags); + if (epf_test->dma_private) { + sconf.direction = dir; + if (dir == DMA_MEM_TO_DEV) + sconf.dst_addr = dma_remote; + else + sconf.src_addr = dma_remote; + + if (dmaengine_slave_config(chan, &sconf)) { + dev_err(dev, "DMA slave config fail\n"); + return -EIO; + } + tx = dmaengine_prep_slave_single(chan, dma_local, len, dir, + flags); + } else { + tx = dmaengine_prep_dma_memcpy(chan, dma_dst, dma_src, len, + flags); + } + if (!tx) { dev_err(dev, "Failed to prepare DMA memcpy\n"); return -EIO; @@ -148,6 +173,23 @@ static int pci_epf_test_data_transfer(struct pci_epf_test *epf_test, return 0; } +struct epf_dma_filter { + struct device *dev; + u32 dma_mask; +}; + +static bool epf_dma_filter_fn(struct dma_chan *chan, void *node) +{ + struct epf_dma_filter *filter = node; + struct dma_slave_caps caps; + + memset(&caps, 0, sizeof(caps)); + dma_get_slave_caps(chan, &caps); + + return chan->device->dev == filter->dev + && (filter->dma_mask & caps.directions); +} + /** * pci_epf_test_init_dma_chan() - Function to initialize EPF test DMA channel * @epf_test: the EPF test device that performs data transfer operation @@ -158,10 +200,44 @@ static int pci_epf_test_init_dma_chan(struct pci_epf_test *epf_test) { struct pci_epf *epf = epf_test->epf; struct device *dev = &epf->dev; + struct epf_dma_filter filter; struct dma_chan *dma_chan; dma_cap_mask_t mask; int ret; + filter.dev = epf->epc->dev.parent; + filter.dma_mask = BIT(DMA_DEV_TO_MEM); + + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + dma_chan = dma_request_channel(mask, epf_dma_filter_fn, &filter); + if (!dma_chan) { + dev_info(dev, "Failed to get private DMA rx channel. Falling back to generic one\n"); + goto fail_back_tx; + } + + epf_test->dma_chan_rx = dma_chan; + + filter.dma_mask = BIT(DMA_MEM_TO_DEV); + dma_chan = dma_request_channel(mask, epf_dma_filter_fn, &filter); + + if (!dma_chan) { + dev_info(dev, "Failed to get private DMA tx channel. Falling back to generic one\n"); + goto fail_back_rx; + } + + epf_test->dma_chan_tx = dma_chan; + epf_test->dma_private = true; + + init_completion(&epf_test->transfer_complete); + + return 0; + +fail_back_rx: + dma_release_channel(epf_test->dma_chan_rx); + epf_test->dma_chan_tx = NULL; + +fail_back_tx: dma_cap_zero(mask); dma_cap_set(DMA_MEMCPY, mask); @@ -174,7 +250,7 @@ static int pci_epf_test_init_dma_chan(struct pci_epf_test *epf_test) } init_completion(&epf_test->transfer_complete); - epf_test->dma_chan = dma_chan; + epf_test->dma_chan_tx = epf_test->dma_chan_rx = dma_chan; return 0; } @@ -190,8 +266,17 @@ static void pci_epf_test_clean_dma_chan(struct pci_epf_test *epf_test) if (!epf_test->dma_supported) return; - dma_release_channel(epf_test->dma_chan); - epf_test->dma_chan = NULL; + dma_release_channel(epf_test->dma_chan_tx); + if (epf_test->dma_chan_tx == epf_test->dma_chan_rx) { + epf_test->dma_chan_tx = NULL; + epf_test->dma_chan_rx = NULL; + return; + } + + dma_release_channel(epf_test->dma_chan_rx); + epf_test->dma_chan_rx = NULL; + + return; } static void pci_epf_test_print_rate(const char *ops, u64 size, @@ -280,8 +365,15 @@ static int pci_epf_test_copy(struct pci_epf_test *epf_test) goto err_map_addr; } + if (epf_test->dma_private) { + dev_err(dev, "Cannot transfer data using DMA\n"); + ret = -EINVAL; + goto err_map_addr; + } + ret = pci_epf_test_data_transfer(epf_test, dst_phys_addr, - src_phys_addr, reg->size); + src_phys_addr, reg->size, 0, + DMA_MEM_TO_MEM); if (ret) dev_err(dev, "Data transfer failed\n"); } else { @@ -373,7 +465,8 @@ static int pci_epf_test_read(struct pci_epf_test *epf_test) ktime_get_ts64(&start); ret = pci_epf_test_data_transfer(epf_test, dst_phys_addr, - phys_addr, reg->size); + phys_addr, reg->size, + reg->src_addr, DMA_DEV_TO_MEM); if (ret) dev_err(dev, "Data transfer failed\n"); ktime_get_ts64(&end); @@ -463,8 +556,11 @@ static int pci_epf_test_write(struct pci_epf_test *epf_test) } ktime_get_ts64(&start); + ret = pci_epf_test_data_transfer(epf_test, phys_addr, - src_phys_addr, reg->size); + src_phys_addr, reg->size, + reg->dst_addr, + DMA_MEM_TO_DEV); if (ret) dev_err(dev, "Data transfer failed\n"); ktime_get_ts64(&end); -- 2.39.5