iommu/amd: Update Device Table in increase_address_space()

The Device Table needs to be updated before the new page-table root
can be published in domain->pt_root. Otherwise a concurrent call to
fetch_pte might fetch a PTE which is not reachable through the Device
Table Entry.

Fixes: 92d420ec028d ("iommu/amd: Relax locking in dma_ops path")
Reported-by: Qian Cai <cai@lca.pw>
Signed-off-by: Joerg Roedel <jroedel@suse.de>
Tested-by: Qian Cai <cai@lca.pw>
Link: https://lore.kernel.org/r/20200504125413.16798-5-joro@8bytes.org
Signed-off-by: Joerg Roedel <jroedel@suse.de>

diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index d2499c86d395c64af14f0c52547811c9aeea61d4..2ae1daac888a081eba76a9b7acdf29b5aae30617 100644 (file)
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -101,6 +101,8 @@ struct kmem_cache *amd_iommu_irq_cache;
 static void update_domain(struct protection_domain *domain);
 static int protection_domain_init(struct protection_domain *domain);
 static void detach_device(struct device *dev);
+static void update_and_flush_device_table(struct protection_domain *domain,
+                                         struct domain_pgtable *pgtable);
 
 /****************************************************************************
  *
@@ -1461,8 +1463,16 @@ static bool increase_address_space(struct protection_domain *domain,
 
        *pte = PM_LEVEL_PDE(pgtable.mode, iommu_virt_to_phys(pgtable.root));
 
-       root = amd_iommu_domain_encode_pgtable(pte, pgtable.mode + 1);
+       pgtable.root  = pte;
+       pgtable.mode += 1;
+       update_and_flush_device_table(domain, &pgtable);
+       domain_flush_complete(domain);
 
+       /*
+        * Device Table needs to be updated and flushed before the new root can
+        * be published.
+        */
+       root = amd_iommu_domain_encode_pgtable(pte, pgtable.mode);
        atomic64_set(&domain->pt_root, root);
 
        ret = true;
@@ -1893,19 +1903,17 @@ static bool dma_ops_domain(struct protection_domain *domain)
 }
 
 static void set_dte_entry(u16 devid, struct protection_domain *domain,
+                         struct domain_pgtable *pgtable,
                          bool ats, bool ppr)
 {
-       struct domain_pgtable pgtable;
        u64 pte_root = 0;
        u64 flags = 0;
        u32 old_domid;
 
-       amd_iommu_domain_get_pgtable(domain, &pgtable);
+       if (pgtable->mode != PAGE_MODE_NONE)
+               pte_root = iommu_virt_to_phys(pgtable->root);
 
-       if (pgtable.mode != PAGE_MODE_NONE)
-               pte_root = iommu_virt_to_phys(pgtable.root);
-
-       pte_root |= (pgtable.mode & DEV_ENTRY_MODE_MASK)
+       pte_root |= (pgtable->mode & DEV_ENTRY_MODE_MASK)
                    << DEV_ENTRY_MODE_SHIFT;
        pte_root |= DTE_FLAG_IR | DTE_FLAG_IW | DTE_FLAG_V | DTE_FLAG_TV;
 
@@ -1978,6 +1986,7 @@ static void clear_dte_entry(u16 devid)
 static void do_attach(struct iommu_dev_data *dev_data,
                      struct protection_domain *domain)
 {
+       struct domain_pgtable pgtable;
        struct amd_iommu *iommu;
        bool ats;
 
@@ -1993,7 +2002,9 @@ static void do_attach(struct iommu_dev_data *dev_data,
        domain->dev_cnt                 += 1;
 
        /* Update device table */
-       set_dte_entry(dev_data->devid, domain, ats, dev_data->iommu_v2);
+       amd_iommu_domain_get_pgtable(domain, &pgtable);
+       set_dte_entry(dev_data->devid, domain, &pgtable,
+                     ats, dev_data->iommu_v2);
        clone_aliases(dev_data->pdev);
 
        device_flush_dte(dev_data);
@@ -2304,22 +2315,34 @@ static int amd_iommu_domain_get_attr(struct iommu_domain *domain,
  *
  *****************************************************************************/
 
-static void update_device_table(struct protection_domain *domain)
+static void update_device_table(struct protection_domain *domain,
+                               struct domain_pgtable *pgtable)
 {
        struct iommu_dev_data *dev_data;
 
        list_for_each_entry(dev_data, &domain->dev_list, list) {
-               set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled,
-                             dev_data->iommu_v2);
+               set_dte_entry(dev_data->devid, domain, pgtable,
+                             dev_data->ats.enabled, dev_data->iommu_v2);
                clone_aliases(dev_data->pdev);
        }
 }
 
+static void update_and_flush_device_table(struct protection_domain *domain,
+                                         struct domain_pgtable *pgtable)
+{
+       update_device_table(domain, pgtable);
+       domain_flush_devices(domain);
+}
+
 static void update_domain(struct protection_domain *domain)
 {
-       update_device_table(domain);
+       struct domain_pgtable pgtable;
 
-       domain_flush_devices(domain);
+       /* Update device table */
+       amd_iommu_domain_get_pgtable(domain, &pgtable);
+       update_and_flush_device_table(domain, &pgtable);
+
+       /* Flush domain TLB(s) and wait for completion */
        domain_flush_tlb_pde(domain);
        domain_flush_complete(domain);
 }