x86: Handle HW IOMMU initialization failure gracefully

If HW IOMMU initialization fails (Intel VT-d often does this,
typically due to BIOS bugs), we fall back to nommu. It doesn't
work for the majority since nowadays we have more than 4GB
memory so we must use swiotlb instead of nommu.

The problem is that it's too late to initialize swiotlb when HW
IOMMU initialization fails. We need to allocate swiotlb memory
earlier from bootmem allocator. Chris explained the issue in
detail:

  http://marc.info/?l=linux-kernel&m=125657444317079&w=2

The current x86 IOMMU initialization sequence is too complicated
and handling the above issue makes it more hacky.

This patch changes x86 IOMMU initialization sequence to handle
the above issue cleanly.

The new x86 IOMMU initialization sequence are:

1. we initialize the swiotlb (and setting swiotlb to 1) in the case
   of (max_pfn > MAX_DMA32_PFN && !no_iommu). dma_ops is set to
   swiotlb_dma_ops or nommu_dma_ops. if swiotlb usage is forced by
   the boot option, we finish here.

2. we call the detection functions of all the IOMMUs

3. the detection function sets x86_init.iommu.iommu_init to the
   IOMMU initialization function (so we can avoid calling the
   initialization functions of all the IOMMUs needlessly).

4. if the IOMMU initialization function doesn't need to swiotlb
   then sets swiotlb to zero (e.g. the initialization is
   sucessful).

5. if we find that swiotlb is set to zero, we free swiotlb
   resource.

Signed-off-by: FUJITA Tomonori <fujita.tomonori@lab.ntt.co.jp>
Cc: chrisw@sous-sol.org
Cc: dwmw2@infradead.org
Cc: joerg.roedel@amd.com
Cc: muli@il.ibm.com
LKML-Reference: <1257849980-22640-10-git-send-email-fujita.tomonori@lab.ntt.co.jp>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/x86/include/asm/iommu.h b/arch/x86/include/asm/iommu.h
index 878b307157665e60b3ff6a0005add8e5b4fe5add..df42a712361f0c304c816b79f8ca85366a04aa23 100644 (file)
--- a/arch/x86/include/asm/iommu.h
+++ b/arch/x86/include/asm/iommu.h
@@ -2,7 +2,6 @@
 #define _ASM_X86_IOMMU_H
 
 static inline void iommu_shutdown_noop(void) {}
-extern void no_iommu_init(void);
 extern struct dma_map_ops nommu_dma_ops;
 extern int force_iommu, no_iommu;
 extern int iommu_detected;

diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index 0285521e0a991d09dbf3fd01e85489f068083d3f..66237fde758fd020d66230c987c510200a82bc10 100644 (file)
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -2110,8 +2110,8 @@ int __init amd_iommu_init_dma_ops(void)
                prealloc_protection_domains();
 
        iommu_detected = 1;
-       force_iommu = 1;
        bad_dma_address = 0;
+       swiotlb = 0;
 #ifdef CONFIG_GART_IOMMU
        gart_iommu_aperture_disabled = 1;
        gart_iommu_aperture = 0;

diff --git a/arch/x86/kernel/amd_iommu_init.c b/arch/x86/kernel/amd_iommu_init.c
index c41aabddaa2a511a9691b6fbc238323c97e4568b..0d4581e602a42858ee23a384d305ecf6a54cb375 100644 (file)
--- a/arch/x86/kernel/amd_iommu_init.c
+++ b/arch/x86/kernel/amd_iommu_init.c
@@ -1330,7 +1330,7 @@ static int __init early_amd_iommu_detect(struct acpi_table_header *table)
 
 void __init amd_iommu_detect(void)
 {
-       if (swiotlb || no_iommu || (iommu_detected && !gart_iommu_aperture))
+       if (no_iommu || (iommu_detected && !gart_iommu_aperture))
                return;
 
        if (acpi_table_parse("IVRS", early_amd_iommu_detect) == 0) {

diff --git a/arch/x86/kernel/aperture_64.c b/arch/x86/kernel/aperture_64.c
index 03933cf0b63ced2dfcb3a119588653933275bc06..e0dfb6856aa297ef9349dc36c3b5cef93b5abbf0 100644 (file)
--- a/arch/x86/kernel/aperture_64.c
+++ b/arch/x86/kernel/aperture_64.c
@@ -458,7 +458,7 @@ out:
 
        if (aper_alloc) {
                /* Got the aperture from the AGP bridge */
-       } else if (swiotlb && !valid_agp) {
+       } else if (!valid_agp) {
                /* Do nothing */
        } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) ||
                   force_iommu ||

diff --git a/arch/x86/kernel/pci-calgary_64.c b/arch/x86/kernel/pci-calgary_64.c
index 47bd419ea4d23b50e0f1b671a66061a01c1abe78..833f491440b9c6b9cab7fffde007afe29f63669f 100644 (file)
--- a/arch/x86/kernel/pci-calgary_64.c
+++ b/arch/x86/kernel/pci-calgary_64.c
@@ -1360,7 +1360,7 @@ void __init detect_calgary(void)
         * if the user specified iommu=off or iommu=soft or we found
         * another HW IOMMU already, bail out.
         */
-       if (swiotlb || no_iommu || iommu_detected)
+       if (no_iommu || iommu_detected)
                return;
 
        if (!use_calgary)
@@ -1445,10 +1445,6 @@ void __init detect_calgary(void)
                printk(KERN_INFO "PCI-DMA: Calgary TCE table spec is %d\n",
                       specified_table_size);
 
-               /* swiotlb for devices that aren't behind the Calgary. */
-               if (max_pfn > MAX_DMA32_PFN)
-                       swiotlb = 1;
-
                x86_init.iommu.iommu_init = calgary_iommu_init;
        }
        return;
@@ -1476,11 +1472,7 @@ int __init calgary_iommu_init(void)
                return ret;
        }
 
-       force_iommu = 1;
        bad_dma_address = 0x0;
-       /* dma_ops is set to swiotlb or nommu */
-       if (!dma_ops)
-               dma_ops = &nommu_dma_ops;
 
        return 0;
 }

diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index bed05e2e589052d5f1f1c7e7b16f60d8b1d500ca..a234e63c26565709dde5f9cb3e79ac5e9bca77b4 100644 (file)
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -124,24 +124,24 @@ static void __init dma32_free_bootmem(void)
 
 void __init pci_iommu_alloc(void)
 {
+       /* swiotlb is forced by the boot option */
+       int use_swiotlb = swiotlb;
 #ifdef CONFIG_X86_64
        /* free the range so iommu could get some range less than 4G */
        dma32_free_bootmem();
 #endif
+       pci_swiotlb_init();
+       if (use_swiotlb)
+               return;
 
-       /*
-        * The order of these functions is important for
-        * fall-back/fail-over reasons
-        */
        gart_iommu_hole_init();
 
        detect_calgary();
 
        detect_intel_iommu();
 
+       /* needs to be called after gart_iommu_hole_init */
        amd_iommu_detect();
-
-       pci_swiotlb_init();
 }
 
 void *dma_generic_alloc_coherent(struct device *dev, size_t size,
@@ -291,10 +291,15 @@ static int __init pci_iommu_init(void)
 #ifdef CONFIG_PCI
        dma_debug_add_bus(&pci_bus_type);
 #endif
-
        x86_init.iommu.iommu_init();
 
-       no_iommu_init();
+       if (swiotlb) {
+               printk(KERN_INFO "PCI-DMA: "
+                      "Using software bounce buffering for IO (SWIOTLB)\n");
+               swiotlb_print_info();
+       } else
+               swiotlb_free();
+
        return 0;
 }
 /* Must execute after PCI subsystem */

diff --git a/arch/x86/kernel/pci-gart_64.c b/arch/x86/kernel/pci-gart_64.c
index 0410bd30060d1ac19bad2e1f87d127c6d32c6ddb..919182e15d1e125124d3a4dd46eb5f07f199e589 100644 (file)
--- a/arch/x86/kernel/pci-gart_64.c
+++ b/arch/x86/kernel/pci-gart_64.c
@@ -833,6 +833,7 @@ int __init gart_iommu_init(void)
        flush_gart();
        dma_ops = &gart_dma_ops;
        x86_platform.iommu_shutdown = gart_iommu_shutdown;
+       swiotlb = 0;
 
        return 0;
 }

diff --git a/arch/x86/kernel/pci-nommu.c b/arch/x86/kernel/pci-nommu.c
index a3933d4330cd31f1ec4c6797f5e892279d9d6c7f..875e3822ae61c27a0e6fa46f857887e7bef4400f 100644 (file)
--- a/arch/x86/kernel/pci-nommu.c
+++ b/arch/x86/kernel/pci-nommu.c
@@ -103,12 +103,3 @@ struct dma_map_ops nommu_dma_ops = {
        .sync_sg_for_device     = nommu_sync_sg_for_device,
        .is_phys                = 1,
 };
-
-void __init no_iommu_init(void)
-{
-       if (dma_ops)
-               return;
-
-       force_iommu = 0; /* no HW IOMMU */
-       dma_ops = &nommu_dma_ops;
-}

diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c
index ea20ef7ca523371e66382e24f794ebce55f2bbc1..17ce4221bd030ca0ca9e43c2e3251201425b2d90 100644 (file)
--- a/arch/x86/kernel/pci-swiotlb.c
+++ b/arch/x86/kernel/pci-swiotlb.c
@@ -46,13 +46,12 @@ void __init pci_swiotlb_init(void)
 {
        /* don't initialize swiotlb if iommu=off (no_iommu=1) */
 #ifdef CONFIG_X86_64
-       if ((!iommu_detected && !no_iommu && max_pfn > MAX_DMA32_PFN))
+       if (!no_iommu && max_pfn > MAX_DMA32_PFN)
                swiotlb = 1;
 #endif
-       if (swiotlb_force)
-               swiotlb = 1;
        if (swiotlb) {
                swiotlb_init(0);
                dma_ops = &swiotlb_dma_ops;
-       }
+       } else
+               dma_ops = &nommu_dma_ops;
 }

diff --git a/drivers/pci/dmar.c b/drivers/pci/dmar.c
index bce9cd7c755a81b8a35d65dcd7a6b5723a854107..437399667e5afb093f965a930f2cc67f3c89faf0 100644 (file)
--- a/drivers/pci/dmar.c
+++ b/drivers/pci/dmar.c
@@ -613,8 +613,7 @@ void __init detect_intel_iommu(void)
                               "x2apic and Intr-remapping.\n");
 #endif
 #ifdef CONFIG_DMAR
-               if (ret && !no_iommu && !iommu_detected && !swiotlb &&
-                   !dmar_disabled)
+               if (ret && !no_iommu && !iommu_detected && !dmar_disabled)
                        iommu_detected = 1;
 #endif
 #ifdef CONFIG_X86

diff --git a/drivers/pci/intel-iommu.c b/drivers/pci/intel-iommu.c
index b1e97e6825009295088d62b183f8136616bbe917..43d755a2e14aa8908a22a3ff54c046b86794c4ff 100644 (file)
--- a/drivers/pci/intel-iommu.c
+++ b/drivers/pci/intel-iommu.c
@@ -3231,7 +3231,7 @@ int __init intel_iommu_init(void)
         * Check the need for DMA-remapping initialization now.
         * Above initialization will also be used by Interrupt-remapping.
         */
-       if (no_iommu || swiotlb || dmar_disabled)
+       if (no_iommu || dmar_disabled)
                return -ENODEV;
 
        iommu_init_mempool();
@@ -3252,7 +3252,9 @@ int __init intel_iommu_init(void)
        "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
 
        init_timer(&unmap_timer);
-       force_iommu = 1;
+#ifdef CONFIG_SWIOTLB
+       swiotlb = 0;
+#endif
        dma_ops = &intel_dma_ops;
 
        init_iommu_sysfs();

diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index 0c12d7cce300d1c8cc465993b3c04e38693fe2d3..e6755a0574fbc3014bf45a7c4bc3cb3c13e81411 100644 (file)
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -109,8 +109,10 @@ setup_io_tlb_npages(char *str)
        }
        if (*str == ',')
                ++str;
-       if (!strcmp(str, "force"))
+       if (!strcmp(str, "force")) {
                swiotlb_force = 1;
+               swiotlb = 1;
+       }
        return 1;
 }
 __setup("swiotlb=", setup_io_tlb_npages);