diff options
Diffstat (limited to 'drivers/iommu/intel-iommu.c')
-rw-r--r-- | drivers/iommu/intel-iommu.c | 281 |
1 files changed, 222 insertions, 59 deletions
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index c621c98c99da..c0c7820d4c46 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -24,6 +24,7 @@ #include <linux/init.h> #include <linux/bitmap.h> #include <linux/debugfs.h> +#include <linux/export.h> #include <linux/slab.h> #include <linux/irq.h> #include <linux/interrupt.h> @@ -306,6 +307,11 @@ static inline bool dma_pte_present(struct dma_pte *pte) return (pte->val & 3) != 0; } +static inline bool dma_pte_superpage(struct dma_pte *pte) +{ + return (pte->val & (1 << 7)); +} + static inline int first_pte_in_page(struct dma_pte *pte) { return !((unsigned long)pte & ~VTD_PAGE_MASK); @@ -393,17 +399,20 @@ static long list_size; static void domain_remove_dev_info(struct dmar_domain *domain); -#ifdef CONFIG_DMAR_DEFAULT_ON +#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON int dmar_disabled = 0; #else int dmar_disabled = 1; -#endif /*CONFIG_DMAR_DEFAULT_ON*/ +#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/ static int dmar_map_gfx = 1; static int dmar_forcedac; static int intel_iommu_strict; static int intel_iommu_superpage = 1; +int intel_iommu_gfx_mapped; +EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped); + #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1)) static DEFINE_SPINLOCK(device_domain_lock); static LIST_HEAD(device_domain_list); @@ -577,17 +586,18 @@ static void domain_update_iommu_snooping(struct dmar_domain *domain) static void domain_update_iommu_superpage(struct dmar_domain *domain) { - int i, mask = 0xf; + struct dmar_drhd_unit *drhd; + struct intel_iommu *iommu = NULL; + int mask = 0xf; if (!intel_iommu_superpage) { domain->iommu_superpage = 0; return; } - domain->iommu_superpage = 4; /* 1TiB */ - - for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) { - mask |= cap_super_page_val(g_iommus[i]->cap); + /* set iommu_superpage to the smallest common denominator */ + for_each_active_iommu(iommu, drhd) { + mask &= cap_super_page_val(iommu->cap); if (!mask) { break; } @@ -730,29 +740,23 @@ out: } static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, - unsigned long pfn, int large_level) + unsigned long pfn, int target_level) { int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT; struct dma_pte *parent, *pte = NULL; int level = agaw_to_level(domain->agaw); - int offset, target_level; + int offset; BUG_ON(!domain->pgd); BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width); parent = domain->pgd; - /* Search pte */ - if (!large_level) - target_level = 1; - else - target_level = large_level; - while (level > 0) { void *tmp_page; offset = pfn_level_offset(pfn, level); pte = &parent[offset]; - if (!large_level && (pte->val & DMA_PTE_LARGE_PAGE)) + if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte))) break; if (level == target_level) break; @@ -816,13 +820,14 @@ static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain, } /* clear last level pte, a tlb flush should be followed */ -static void dma_pte_clear_range(struct dmar_domain *domain, +static int dma_pte_clear_range(struct dmar_domain *domain, unsigned long start_pfn, unsigned long last_pfn) { int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT; unsigned int large_page = 1; struct dma_pte *first_pte, *pte; + int order; BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width); BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width); @@ -846,6 +851,9 @@ static void dma_pte_clear_range(struct dmar_domain *domain, (void *)pte - (void *)first_pte); } while (start_pfn && start_pfn <= last_pfn); + + order = (large_page - 1) * 9; + return order; } /* free page table pages. last level pte should already be cleared */ @@ -932,7 +940,7 @@ static void iommu_set_root_entry(struct intel_iommu *iommu) addr = iommu->root_entry; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr)); writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG); @@ -941,7 +949,7 @@ static void iommu_set_root_entry(struct intel_iommu *iommu) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_RTPS), sts); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } static void iommu_flush_write_buffer(struct intel_iommu *iommu) @@ -952,14 +960,14 @@ static void iommu_flush_write_buffer(struct intel_iommu *iommu) if (!rwbf_quirk && !cap_rwbf(iommu->cap)) return; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG); /* Make sure hardware complete it */ IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (!(val & DMA_GSTS_WBFS)), val); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } /* return value determine if we need a write buffer flush */ @@ -986,14 +994,14 @@ static void __iommu_flush_context(struct intel_iommu *iommu, } val |= DMA_CCMD_ICC; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); dmar_writeq(iommu->reg + DMAR_CCMD_REG, val); /* Make sure hardware complete it */ IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG, dmar_readq, (!(val & DMA_CCMD_ICC)), val); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } /* return value determine if we need a write buffer flush */ @@ -1032,7 +1040,7 @@ static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, if (cap_write_drain(iommu->cap)) val |= DMA_TLB_WRITE_DRAIN; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); /* Note: Only uses first TLB reg currently */ if (val_iva) dmar_writeq(iommu->reg + tlb_offset, val_iva); @@ -1042,7 +1050,7 @@ static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, IOMMU_WAIT_OP(iommu, tlb_offset + 8, dmar_readq, (!(val & DMA_TLB_IVT)), val); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); /* check IOTLB invalidation granularity */ if (DMA_TLB_IAIG(val) == 0) @@ -1158,7 +1166,7 @@ static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu) u32 pmen; unsigned long flags; - spin_lock_irqsave(&iommu->register_lock, flags); + raw_spin_lock_irqsave(&iommu->register_lock, flags); pmen = readl(iommu->reg + DMAR_PMEN_REG); pmen &= ~DMA_PMEN_EPM; writel(pmen, iommu->reg + DMAR_PMEN_REG); @@ -1167,7 +1175,7 @@ static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu) IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG, readl, !(pmen & DMA_PMEN_PRS), pmen); - spin_unlock_irqrestore(&iommu->register_lock, flags); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); } static int iommu_enable_translation(struct intel_iommu *iommu) @@ -1175,7 +1183,7 @@ static int iommu_enable_translation(struct intel_iommu *iommu) u32 sts; unsigned long flags; - spin_lock_irqsave(&iommu->register_lock, flags); + raw_spin_lock_irqsave(&iommu->register_lock, flags); iommu->gcmd |= DMA_GCMD_TE; writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); @@ -1183,7 +1191,7 @@ static int iommu_enable_translation(struct intel_iommu *iommu) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_TES), sts); - spin_unlock_irqrestore(&iommu->register_lock, flags); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); return 0; } @@ -1192,7 +1200,7 @@ static int iommu_disable_translation(struct intel_iommu *iommu) u32 sts; unsigned long flag; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); iommu->gcmd &= ~DMA_GCMD_TE; writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); @@ -1200,7 +1208,7 @@ static int iommu_disable_translation(struct intel_iommu *iommu) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (!(sts & DMA_GSTS_TES)), sts); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); return 0; } @@ -2150,7 +2158,7 @@ static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr, rmrr->end_address); } -#ifdef CONFIG_DMAR_FLOPPY_WA +#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA static inline void iommu_prepare_isa(void) { struct pci_dev *pdev; @@ -2173,7 +2181,7 @@ static inline void iommu_prepare_isa(void) { return; } -#endif /* !CONFIG_DMAR_FLPY_WA */ +#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */ static int md_domain_init(struct dmar_domain *domain, int guest_width); @@ -2484,7 +2492,7 @@ static int __init init_dmars(void) if (iommu_pass_through) iommu_identity_mapping |= IDENTMAP_ALL; -#ifdef CONFIG_DMAR_BROKEN_GFX_WA +#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA iommu_identity_mapping |= IDENTMAP_GFX; #endif @@ -3226,9 +3234,6 @@ static void __init init_no_remapping_devices(void) } } - if (dmar_map_gfx) - return; - for_each_drhd_unit(drhd) { int i; if (drhd->ignored || drhd->include_all) @@ -3236,18 +3241,23 @@ static void __init init_no_remapping_devices(void) for (i = 0; i < drhd->devices_cnt; i++) if (drhd->devices[i] && - !IS_GFX_DEVICE(drhd->devices[i])) + !IS_GFX_DEVICE(drhd->devices[i])) break; if (i < drhd->devices_cnt) continue; - /* bypass IOMMU if it is just for gfx devices */ - drhd->ignored = 1; - for (i = 0; i < drhd->devices_cnt; i++) { - if (!drhd->devices[i]) - continue; - drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO; + /* This IOMMU has *only* gfx devices. Either bypass it or + set the gfx_mapped flag, as appropriate */ + if (dmar_map_gfx) { + intel_iommu_gfx_mapped = 1; + } else { + drhd->ignored = 1; + for (i = 0; i < drhd->devices_cnt; i++) { + if (!drhd->devices[i]) + continue; + drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO; + } } } } @@ -3320,7 +3330,7 @@ static int iommu_suspend(void) for_each_active_iommu(iommu, drhd) { iommu_disable_translation(iommu); - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); iommu->iommu_state[SR_DMAR_FECTL_REG] = readl(iommu->reg + DMAR_FECTL_REG); @@ -3331,7 +3341,7 @@ static int iommu_suspend(void) iommu->iommu_state[SR_DMAR_FEUADDR_REG] = readl(iommu->reg + DMAR_FEUADDR_REG); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } return 0; @@ -3358,7 +3368,7 @@ static void iommu_resume(void) for_each_active_iommu(iommu, drhd) { - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); writel(iommu->iommu_state[SR_DMAR_FECTL_REG], iommu->reg + DMAR_FECTL_REG); @@ -3369,7 +3379,7 @@ static void iommu_resume(void) writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG], iommu->reg + DMAR_FEUADDR_REG); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } for_each_active_iommu(iommu, drhd) @@ -3390,6 +3400,151 @@ static void __init init_iommu_pm_ops(void) static inline void init_iommu_pm_ops(void) {} #endif /* CONFIG_PM */ +LIST_HEAD(dmar_rmrr_units); + +static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr) +{ + list_add(&rmrr->list, &dmar_rmrr_units); +} + + +int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header) +{ + struct acpi_dmar_reserved_memory *rmrr; + struct dmar_rmrr_unit *rmrru; + + rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL); + if (!rmrru) + return -ENOMEM; + + rmrru->hdr = header; + rmrr = (struct acpi_dmar_reserved_memory *)header; + rmrru->base_address = rmrr->base_address; + rmrru->end_address = rmrr->end_address; + + dmar_register_rmrr_unit(rmrru); + return 0; +} + +static int __init +rmrr_parse_dev(struct dmar_rmrr_unit *rmrru) +{ + struct acpi_dmar_reserved_memory *rmrr; + int ret; + + rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr; + ret = dmar_parse_dev_scope((void *)(rmrr + 1), + ((void *)rmrr) + rmrr->header.length, + &rmrru->devices_cnt, &rmrru->devices, rmrr->segment); + + if (ret || (rmrru->devices_cnt == 0)) { + list_del(&rmrru->list); + kfree(rmrru); + } + return ret; +} + +static LIST_HEAD(dmar_atsr_units); + +int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr) +{ + struct acpi_dmar_atsr *atsr; + struct dmar_atsr_unit *atsru; + + atsr = container_of(hdr, struct acpi_dmar_atsr, header); + atsru = kzalloc(sizeof(*atsru), GFP_KERNEL); + if (!atsru) + return -ENOMEM; + + atsru->hdr = hdr; + atsru->include_all = atsr->flags & 0x1; + + list_add(&atsru->list, &dmar_atsr_units); + + return 0; +} + +static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru) +{ + int rc; + struct acpi_dmar_atsr *atsr; + + if (atsru->include_all) + return 0; + + atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header); + rc = dmar_parse_dev_scope((void *)(atsr + 1), + (void *)atsr + atsr->header.length, + &atsru->devices_cnt, &atsru->devices, + atsr->segment); + if (rc || !atsru->devices_cnt) { + list_del(&atsru->list); + kfree(atsru); + } + + return rc; +} + +int dmar_find_matched_atsr_unit(struct pci_dev *dev) +{ + int i; + struct pci_bus *bus; + struct acpi_dmar_atsr *atsr; + struct dmar_atsr_unit *atsru; + + dev = pci_physfn(dev); + + list_for_each_entry(atsru, &dmar_atsr_units, list) { + atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header); + if (atsr->segment == pci_domain_nr(dev->bus)) + goto found; + } + + return 0; + +found: + for (bus = dev->bus; bus; bus = bus->parent) { + struct pci_dev *bridge = bus->self; + + if (!bridge || !pci_is_pcie(bridge) || + bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) + return 0; + + if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) { + for (i = 0; i < atsru->devices_cnt; i++) + if (atsru->devices[i] == bridge) + return 1; + break; + } + } + + if (atsru->include_all) + return 1; + + return 0; +} + +int dmar_parse_rmrr_atsr_dev(void) +{ + struct dmar_rmrr_unit *rmrr, *rmrr_n; + struct dmar_atsr_unit *atsr, *atsr_n; + int ret = 0; + + list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) { + ret = rmrr_parse_dev(rmrr); + if (ret) + return ret; + } + + list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) { + ret = atsr_parse_dev(atsr); + if (ret) + return ret; + } + + return ret; +} + /* * Here we only respond to action of unbound device from driver. * @@ -3439,16 +3594,12 @@ int __init intel_iommu_init(void) return -ENODEV; } - if (dmar_dev_scope_init()) { + if (dmar_dev_scope_init() < 0) { if (force_on) panic("tboot: Failed to initialize DMAR device scope\n"); return -ENODEV; } - /* - * Check the need for DMA-remapping initialization now. - * Above initialization will also be used by Interrupt-remapping. - */ if (no_iommu || dmar_disabled) return -ENODEV; @@ -3458,6 +3609,12 @@ int __init intel_iommu_init(void) return -ENODEV; } + if (list_empty(&dmar_rmrr_units)) + printk(KERN_INFO "DMAR: No RMRR found\n"); + + if (list_empty(&dmar_atsr_units)) + printk(KERN_INFO "DMAR: No ATSR found\n"); + if (dmar_init_reserved_ranges()) { if (force_on) panic("tboot: Failed to reserve iommu ranges\n"); @@ -3486,7 +3643,7 @@ int __init intel_iommu_init(void) init_iommu_pm_ops(); - register_iommu(&intel_iommu_ops); + bus_set_iommu(&pci_bus_type, &intel_iommu_ops); bus_register_notifier(&pci_bus_type, &device_nb); @@ -3568,6 +3725,8 @@ static void domain_remove_one_dev_info(struct dmar_domain *domain, found = 1; } + spin_unlock_irqrestore(&device_domain_lock, flags); + if (found == 0) { unsigned long tmp_flags; spin_lock_irqsave(&domain->iommu_lock, tmp_flags); @@ -3584,8 +3743,6 @@ static void domain_remove_one_dev_info(struct dmar_domain *domain, spin_unlock_irqrestore(&iommu->lock, tmp_flags); } } - - spin_unlock_irqrestore(&device_domain_lock, flags); } static void vm_domain_remove_all_dev_info(struct dmar_domain *domain) @@ -3739,6 +3896,7 @@ static int intel_iommu_domain_init(struct iommu_domain *domain) vm_domain_exit(dmar_domain); return -ENOMEM; } + domain_update_iommu_cap(dmar_domain); domain->priv = dmar_domain; return 0; @@ -3864,14 +4022,15 @@ static int intel_iommu_unmap(struct iommu_domain *domain, { struct dmar_domain *dmar_domain = domain->priv; size_t size = PAGE_SIZE << gfp_order; + int order; - dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT, + order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT, (iova + size - 1) >> VTD_PAGE_SHIFT); if (dmar_domain->max_addr == iova + size) dmar_domain->max_addr = iova; - return gfp_order; + return order; } static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, @@ -3950,7 +4109,11 @@ static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev) if (!(ggc & GGC_MEMORY_VT_ENABLED)) { printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n"); dmar_map_gfx = 0; - } + } else if (dmar_map_gfx) { + /* we have to ensure the gfx device is idle before we flush */ + printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n"); + intel_iommu_strict = 1; + } } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt); |