KVM: PPC: Optimize clearing TCEs for sparse tables

The powernv platform maintains 2 TCE tables for VFIO - a hardware TCE
table and a table with userspace addresses. These tables are radix trees,
we allocate indirect levels when they are written to. Since
the memory allocation is problematic in real mode, we have 2 accessors
to the entries:
- for virtual mode: it allocates the memory and it is always expected
to return non-NULL;
- fr real mode: it does not allocate and can return NULL.

Also, DMA windows can span to up to 55 bits of the address space and since
we never have this much RAM, such windows are sparse. However currently
the SPAPR TCE IOMMU driver walks through all TCEs to unpin DMA memory.

Since we maintain a userspace addresses table for VFIO which is a mirror
of the hardware table, we can use it to know which parts of the DMA
window have not been mapped and skip these so does this patch.

The bare metal systems do not have this problem as they use a bypass mode
of a PHB which maps RAM directly.

This helps a lot with sparse DMA windows, reducing the shutdown time from
about 3 minutes per 1 billion TCEs to a few seconds for 32GB sparse guest.
Just skipping the last level seems to be good enough.

As non-allocating accessor is used now in virtual mode as well, rename it
from IOMMU_TABLE_USERSPACE_ENTRY_RM (real mode) to _RO (read only).

Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: Paul Mackerras <paulus@ozlabs.org>

diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h
index 3d4b88cb859924cf2fe438a49c7be2e6febbaa12..35db0cbc92227723ed5462a526bc3f68f5b15fe4 100644 (file)
--- a/arch/powerpc/include/asm/iommu.h
+++ b/arch/powerpc/include/asm/iommu.h
@@ -126,7 +126,7 @@ struct iommu_table {
        int it_nid;
 };
 
-#define IOMMU_TABLE_USERSPACE_ENTRY_RM(tbl, entry) \
+#define IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry) \
                ((tbl)->it_ops->useraddrptr((tbl), (entry), false))
 #define IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry) \
                ((tbl)->it_ops->useraddrptr((tbl), (entry), true))

diff --git a/arch/powerpc/kvm/book3s_64_vio.c b/arch/powerpc/kvm/book3s_64_vio.c
index c0c64d11cc7131bfb2ab206d18cd64fd1ea2ecae..62a8d03ba7e96e1cfc012abc5452192c5cc005ec 100644 (file)
--- a/arch/powerpc/kvm/book3s_64_vio.c
+++ b/arch/powerpc/kvm/book3s_64_vio.c
@@ -410,11 +410,10 @@ static long kvmppc_tce_iommu_mapped_dec(struct kvm *kvm,
 {
        struct mm_iommu_table_group_mem_t *mem = NULL;
        const unsigned long pgsize = 1ULL << tbl->it_page_shift;
-       __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry);
+       __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);
 
        if (!pua)
-               /* it_userspace allocation might be delayed */
-               return H_TOO_HARD;
+               return H_SUCCESS;
 
        mem = mm_iommu_lookup(kvm->mm, be64_to_cpu(*pua), pgsize);
        if (!mem)

diff --git a/arch/powerpc/kvm/book3s_64_vio_hv.c b/arch/powerpc/kvm/book3s_64_vio_hv.c
index ec99363fdf54a57f208d24a12918cab0313c8059..2206bc729b9a07c4c0715af3fbabc916f7e13a72 100644 (file)
--- a/arch/powerpc/kvm/book3s_64_vio_hv.c
+++ b/arch/powerpc/kvm/book3s_64_vio_hv.c
@@ -214,7 +214,7 @@ static long iommu_tce_xchg_rm(struct mm_struct *mm, struct iommu_table *tbl,
 
        if (!ret && ((*direction == DMA_FROM_DEVICE) ||
                                (*direction == DMA_BIDIRECTIONAL))) {
-               __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RM(tbl, entry);
+               __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);
                /*
                 * kvmppc_rm_tce_iommu_do_map() updates the UA cache after
                 * calling this so we still get here a valid UA.
@@ -240,7 +240,7 @@ static long kvmppc_rm_tce_iommu_mapped_dec(struct kvm *kvm,
 {
        struct mm_iommu_table_group_mem_t *mem = NULL;
        const unsigned long pgsize = 1ULL << tbl->it_page_shift;
-       __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RM(tbl, entry);
+       __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);
 
        if (!pua)
                /* it_userspace allocation might be delayed */
@@ -304,7 +304,7 @@ static long kvmppc_rm_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl,
 {
        long ret;
        unsigned long hpa = 0;
-       __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RM(tbl, entry);
+       __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);
        struct mm_iommu_table_group_mem_t *mem;
 
        if (!pua)

diff --git a/drivers/vfio/vfio_iommu_spapr_tce.c b/drivers/vfio/vfio_iommu_spapr_tce.c
index 96721b154454f617689e56ccc029eac9181e4b85..b30926e11d8789e88204196f957112c0f13693e9 100644 (file)
--- a/drivers/vfio/vfio_iommu_spapr_tce.c
+++ b/drivers/vfio/vfio_iommu_spapr_tce.c
@@ -444,7 +444,7 @@ static void tce_iommu_unuse_page_v2(struct tce_container *container,
        struct mm_iommu_table_group_mem_t *mem = NULL;
        int ret;
        unsigned long hpa = 0;
-       __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry);
+       __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);
 
        if (!pua)
                return;
@@ -467,8 +467,27 @@ static int tce_iommu_clear(struct tce_container *container,
        unsigned long oldhpa;
        long ret;
        enum dma_data_direction direction;
+       unsigned long lastentry = entry + pages;
+
+       for ( ; entry < lastentry; ++entry) {
+               if (tbl->it_indirect_levels && tbl->it_userspace) {
+                       /*
+                        * For multilevel tables, we can take a shortcut here
+                        * and skip some TCEs as we know that the userspace
+                        * addresses cache is a mirror of the real TCE table
+                        * and if it is missing some indirect levels, then
+                        * the hardware table does not have them allocated
+                        * either and therefore does not require updating.
+                        */
+                       __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl,
+                                       entry);
+                       if (!pua) {
+                               /* align to level_size which is power of two */
+                               entry |= tbl->it_level_size - 1;
+                               continue;
+                       }
+               }
 
-       for ( ; pages; --pages, ++entry) {
                cond_resched();
 
                direction = DMA_NONE;