From 31e6850e0fdb3a586363cc4d2f9801cdf9374310 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 5 Apr 2016 12:39:30 +0100 Subject: iommu: Add MMIO mapping type On some platforms, MMIO regions might need slightly different treatment compared to mapping regular memory; add the notion of MMIO mappings to the IOMMU API's memory type flags, so that callers can let the IOMMU drivers know to do the right thing. Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- include/linux/iommu.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/iommu.h b/include/linux/iommu.h index a5c539fa5d2b..34b643227df1 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -30,6 +30,7 @@ #define IOMMU_WRITE (1 << 1) #define IOMMU_CACHE (1 << 2) /* DMA cache coherency */ #define IOMMU_NOEXEC (1 << 3) +#define IOMMU_MMIO (1 << 4) /* e.g. things like MSI doorbells */ struct iommu_ops; struct iommu_group; -- cgit v1.2.3 From e511267bc25e18926826e7cccdf7872bcbb4776a Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 26 Apr 2016 11:38:20 +0100 Subject: io-64-nonatomic: Add relaxed accessor variants Whilst commit 9439eb3ab9d1 ("asm-generic: io: implement relaxed accessor macros as conditional wrappers") makes the *_relaxed forms of I/O accessors universally available to drivers, in cases where writeq() is implemented via the io-64-nonatomic helpers, writeq_relaxed() will end up falling back to writel() regardless of whether writel_relaxed() is available (identically for s/write/read/). Add corresponding relaxed forms of the nonatomic helpers to delegate to the equivalent 32-bit accessors as appropriate. We also need to fix io.h to avoid defining default relaxed variants if the basic accessors themselves don't exist. CC: Christoph Hellwig CC: Darren Hart CC: Hitoshi Mitake Acked-by: Arnd Bergmann Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- include/asm-generic/io.h | 4 ++-- include/linux/io-64-nonatomic-hi-lo.h | 25 +++++++++++++++++++++++++ include/linux/io-64-nonatomic-lo-hi.h | 25 +++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h index eed3bbe88c8a..002b81f6f2bc 100644 --- a/include/asm-generic/io.h +++ b/include/asm-generic/io.h @@ -191,7 +191,7 @@ static inline void writeq(u64 value, volatile void __iomem *addr) #define readl_relaxed readl #endif -#ifndef readq_relaxed +#if defined(readq) && !defined(readq_relaxed) #define readq_relaxed readq #endif @@ -207,7 +207,7 @@ static inline void writeq(u64 value, volatile void __iomem *addr) #define writel_relaxed writel #endif -#ifndef writeq_relaxed +#if defined(writeq) && !defined(writeq_relaxed) #define writeq_relaxed writeq #endif diff --git a/include/linux/io-64-nonatomic-hi-lo.h b/include/linux/io-64-nonatomic-hi-lo.h index 11d7e840d913..defcc4644ce3 100644 --- a/include/linux/io-64-nonatomic-hi-lo.h +++ b/include/linux/io-64-nonatomic-hi-lo.h @@ -21,6 +21,23 @@ static inline void hi_lo_writeq(__u64 val, volatile void __iomem *addr) writel(val, addr); } +static inline __u64 hi_lo_readq_relaxed(const volatile void __iomem *addr) +{ + const volatile u32 __iomem *p = addr; + u32 low, high; + + high = readl_relaxed(p + 1); + low = readl_relaxed(p); + + return low + ((u64)high << 32); +} + +static inline void hi_lo_writeq_relaxed(__u64 val, volatile void __iomem *addr) +{ + writel_relaxed(val >> 32, addr + 4); + writel_relaxed(val, addr); +} + #ifndef readq #define readq hi_lo_readq #endif @@ -29,4 +46,12 @@ static inline void hi_lo_writeq(__u64 val, volatile void __iomem *addr) #define writeq hi_lo_writeq #endif +#ifndef readq_relaxed +#define readq_relaxed hi_lo_readq_relaxed +#endif + +#ifndef writeq_relaxed +#define writeq_relaxed hi_lo_writeq_relaxed +#endif + #endif /* _LINUX_IO_64_NONATOMIC_HI_LO_H_ */ diff --git a/include/linux/io-64-nonatomic-lo-hi.h b/include/linux/io-64-nonatomic-lo-hi.h index 1a4315f97360..084461a4e5ab 100644 --- a/include/linux/io-64-nonatomic-lo-hi.h +++ b/include/linux/io-64-nonatomic-lo-hi.h @@ -21,6 +21,23 @@ static inline void lo_hi_writeq(__u64 val, volatile void __iomem *addr) writel(val >> 32, addr + 4); } +static inline __u64 lo_hi_readq_relaxed(const volatile void __iomem *addr) +{ + const volatile u32 __iomem *p = addr; + u32 low, high; + + low = readl_relaxed(p); + high = readl_relaxed(p + 1); + + return low + ((u64)high << 32); +} + +static inline void lo_hi_writeq_relaxed(__u64 val, volatile void __iomem *addr) +{ + writel_relaxed(val, addr); + writel_relaxed(val >> 32, addr + 4); +} + #ifndef readq #define readq lo_hi_readq #endif @@ -29,4 +46,12 @@ static inline void lo_hi_writeq(__u64 val, volatile void __iomem *addr) #define writeq lo_hi_writeq #endif +#ifndef readq_relaxed +#define readq_relaxed lo_hi_readq_relaxed +#endif + +#ifndef writeq_relaxed +#define writeq_relaxed lo_hi_writeq_relaxed +#endif + #endif /* _LINUX_IO_64_NONATOMIC_LO_HI_H_ */ -- cgit v1.2.3 From 3c3e8943ac6f36ca5d18ca61b30634fb560b4ebb Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 7 Apr 2016 18:42:04 +0100 Subject: iommu: remove unused priv field from struct iommu_ops The priv field from iommu_ops is a hangover from the of_dma_configure series and isn't actually used. Remove it before it has chance to spread. Signed-off-by: Will Deacon Acked-by: Laurent Pinchart Acked-by: Will Deacon Signed-off-by: Joerg Roedel --- include/linux/iommu.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/iommu.h b/include/linux/iommu.h index ef7a6ecd8584..8a2570443b80 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -156,7 +156,6 @@ struct iommu_dm_region { * @domain_get_windows: Return the number of windows for a domain * @of_xlate: add OF master IDs to iommu grouping * @pgsize_bitmap: bitmap of supported page sizes - * @priv: per-instance data private to the iommu driver */ struct iommu_ops { bool (*capable)(enum iommu_cap); @@ -198,7 +197,6 @@ struct iommu_ops { int (*of_xlate)(struct device *dev, struct of_phandle_args *args); unsigned long pgsize_bitmap; - void *priv; }; #define IOMMU_GROUP_NOTIFY_ADD_DEVICE 1 /* Device added */ -- cgit v1.2.3 From 53c92d793395fdab9edbd2f79b084bb6b2e6ae79 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 7 Apr 2016 18:42:05 +0100 Subject: iommu: of: enforce const-ness of struct iommu_ops As a set of driver-provided callbacks and static data, there is no compelling reason for struct iommu_ops to be mutable in core code, so enforce const-ness throughout. Acked-by: Thierry Reding Signed-off-by: Robin Murphy Acked-by: Will Deacon Signed-off-by: Joerg Roedel --- arch/arm/include/asm/dma-mapping.h | 2 +- arch/arm/mm/dma-mapping.c | 6 +++--- arch/arm64/include/asm/dma-mapping.h | 2 +- arch/arm64/mm/dma-mapping.c | 4 ++-- drivers/iommu/of_iommu.c | 14 +++++++------- drivers/of/device.c | 2 +- include/linux/dma-mapping.h | 2 +- include/linux/of_iommu.h | 8 ++++---- 8 files changed, 20 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index 6ad1ceda62a5..02283eb2f5b2 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -118,7 +118,7 @@ static inline unsigned long dma_max_pfn(struct device *dev) #define arch_setup_dma_ops arch_setup_dma_ops extern void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - struct iommu_ops *iommu, bool coherent); + const struct iommu_ops *iommu, bool coherent); #define arch_teardown_dma_ops arch_teardown_dma_ops extern void arch_teardown_dma_ops(struct device *dev); diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index c941e93048ad..5c2ca062c3fa 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -2215,7 +2215,7 @@ static struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent) } static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, - struct iommu_ops *iommu) + const struct iommu_ops *iommu) { struct dma_iommu_mapping *mapping; @@ -2253,7 +2253,7 @@ static void arm_teardown_iommu_dma_ops(struct device *dev) #else static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, - struct iommu_ops *iommu) + const struct iommu_ops *iommu) { return false; } @@ -2270,7 +2270,7 @@ static struct dma_map_ops *arm_get_dma_map_ops(bool coherent) } void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - struct iommu_ops *iommu, bool coherent) + const struct iommu_ops *iommu, bool coherent) { struct dma_map_ops *dma_ops; diff --git a/arch/arm64/include/asm/dma-mapping.h b/arch/arm64/include/asm/dma-mapping.h index ba437f090a74..7dbea6c070ec 100644 --- a/arch/arm64/include/asm/dma-mapping.h +++ b/arch/arm64/include/asm/dma-mapping.h @@ -48,7 +48,7 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) } void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - struct iommu_ops *iommu, bool coherent); + const struct iommu_ops *iommu, bool coherent); #define arch_setup_dma_ops arch_setup_dma_ops #ifdef CONFIG_IOMMU_DMA diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index a6e757cbab77..5d36907f9b12 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -979,13 +979,13 @@ void arch_teardown_dma_ops(struct device *dev) #else static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - struct iommu_ops *iommu) + const struct iommu_ops *iommu) { } #endif /* CONFIG_IOMMU_DMA */ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - struct iommu_ops *iommu, bool coherent) + const struct iommu_ops *iommu, bool coherent) { if (!dev->archdata.dma_ops) dev->archdata.dma_ops = &swiotlb_dma_ops; diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 5fea665af99d..af499aea0a1a 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -98,12 +98,12 @@ EXPORT_SYMBOL_GPL(of_get_dma_window); struct of_iommu_node { struct list_head list; struct device_node *np; - struct iommu_ops *ops; + const struct iommu_ops *ops; }; static LIST_HEAD(of_iommu_list); static DEFINE_SPINLOCK(of_iommu_lock); -void of_iommu_set_ops(struct device_node *np, struct iommu_ops *ops) +void of_iommu_set_ops(struct device_node *np, const struct iommu_ops *ops) { struct of_iommu_node *iommu = kzalloc(sizeof(*iommu), GFP_KERNEL); @@ -119,10 +119,10 @@ void of_iommu_set_ops(struct device_node *np, struct iommu_ops *ops) spin_unlock(&of_iommu_lock); } -struct iommu_ops *of_iommu_get_ops(struct device_node *np) +const struct iommu_ops *of_iommu_get_ops(struct device_node *np) { struct of_iommu_node *node; - struct iommu_ops *ops = NULL; + const struct iommu_ops *ops = NULL; spin_lock(&of_iommu_lock); list_for_each_entry(node, &of_iommu_list, list) @@ -134,12 +134,12 @@ struct iommu_ops *of_iommu_get_ops(struct device_node *np) return ops; } -struct iommu_ops *of_iommu_configure(struct device *dev, - struct device_node *master_np) +const struct iommu_ops *of_iommu_configure(struct device *dev, + struct device_node *master_np) { struct of_phandle_args iommu_spec; struct device_node *np; - struct iommu_ops *ops = NULL; + const struct iommu_ops *ops = NULL; int idx = 0; /* diff --git a/drivers/of/device.c b/drivers/of/device.c index e5f47cec75f3..fd5cfad7c403 100644 --- a/drivers/of/device.c +++ b/drivers/of/device.c @@ -88,7 +88,7 @@ void of_dma_configure(struct device *dev, struct device_node *np) int ret; bool coherent; unsigned long offset; - struct iommu_ops *iommu; + const struct iommu_ops *iommu; /* * Set default coherent_dma_mask to 32 bit. Drivers are expected to diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 9ea9aba28049..71c1b215ef66 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -514,7 +514,7 @@ extern u64 dma_get_required_mask(struct device *dev); #ifndef arch_setup_dma_ops static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base, - u64 size, struct iommu_ops *iommu, + u64 size, const struct iommu_ops *iommu, bool coherent) { } #endif diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h index ffbe4707d4aa..bd02b44902d0 100644 --- a/include/linux/of_iommu.h +++ b/include/linux/of_iommu.h @@ -12,7 +12,7 @@ extern int of_get_dma_window(struct device_node *dn, const char *prefix, size_t *size); extern void of_iommu_init(void); -extern struct iommu_ops *of_iommu_configure(struct device *dev, +extern const struct iommu_ops *of_iommu_configure(struct device *dev, struct device_node *master_np); #else @@ -25,7 +25,7 @@ static inline int of_get_dma_window(struct device_node *dn, const char *prefix, } static inline void of_iommu_init(void) { } -static inline struct iommu_ops *of_iommu_configure(struct device *dev, +static inline const struct iommu_ops *of_iommu_configure(struct device *dev, struct device_node *master_np) { return NULL; @@ -33,8 +33,8 @@ static inline struct iommu_ops *of_iommu_configure(struct device *dev, #endif /* CONFIG_OF_IOMMU */ -void of_iommu_set_ops(struct device_node *np, struct iommu_ops *ops); -struct iommu_ops *of_iommu_get_ops(struct device_node *np); +void of_iommu_set_ops(struct device_node *np, const struct iommu_ops *ops); +const struct iommu_ops *of_iommu_get_ops(struct device_node *np); extern struct of_device_id __iommu_of_table; -- cgit v1.2.3 From d16e0faab911cc0e100a1e8e93635b432566608e Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 7 Apr 2016 18:42:06 +0100 Subject: iommu: Allow selecting page sizes per domain Many IOMMUs support multiple page table formats, meaning that any given domain may only support a subset of the hardware page sizes presented in iommu_ops->pgsize_bitmap. There are also certain use-cases where the creator of a domain may want to control which page sizes are used, for example to force the use of hugepage mappings to reduce pagetable walk depth. To this end, add a per-domain pgsize_bitmap to represent the subset of page sizes actually in use, to make it possible for domains with different requirements to coexist. Signed-off-by: Will Deacon [rm: hijacked and rebased original patch with new commit message] Signed-off-by: Robin Murphy Acked-by: Will Deacon Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 2 +- drivers/iommu/iommu.c | 22 ++++++++++++---------- drivers/iommu/mtk_iommu.c | 2 +- drivers/vfio/vfio_iommu_type1.c | 2 +- include/linux/iommu.h | 3 ++- 5 files changed, 17 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 886cb3a78326..99432999b52f 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -94,7 +94,7 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, u64 size return -ENODEV; /* Use the smallest supported page size for IOVA granularity */ - order = __ffs(domain->ops->pgsize_bitmap); + order = __ffs(domain->pgsize_bitmap); base_pfn = max_t(unsigned long, 1, base >> order); end_pfn = (base + size - 1) >> order; diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index b9df1411c894..ab4d014e3687 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -337,9 +337,9 @@ static int iommu_group_create_direct_mappings(struct iommu_group *group, if (!domain || domain->type != IOMMU_DOMAIN_DMA) return 0; - BUG_ON(!domain->ops->pgsize_bitmap); + BUG_ON(!domain->pgsize_bitmap); - pg_size = 1UL << __ffs(domain->ops->pgsize_bitmap); + pg_size = 1UL << __ffs(domain->pgsize_bitmap); INIT_LIST_HEAD(&mappings); iommu_get_dm_regions(dev, &mappings); @@ -1073,6 +1073,8 @@ static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, domain->ops = bus->iommu_ops; domain->type = type; + /* Assume all sizes by default; the driver may override this later */ + domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap; return domain; } @@ -1297,7 +1299,7 @@ static size_t iommu_pgsize(struct iommu_domain *domain, pgsize = (1UL << (pgsize_idx + 1)) - 1; /* throw away page sizes not supported by the hardware */ - pgsize &= domain->ops->pgsize_bitmap; + pgsize &= domain->pgsize_bitmap; /* make sure we're still sane */ BUG_ON(!pgsize); @@ -1319,14 +1321,14 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova, int ret = 0; if (unlikely(domain->ops->map == NULL || - domain->ops->pgsize_bitmap == 0UL)) + domain->pgsize_bitmap == 0UL)) return -ENODEV; if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) return -EINVAL; /* find out the minimum page size supported */ - min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap); + min_pagesz = 1 << __ffs(domain->pgsize_bitmap); /* * both the virtual address and the physical one, as well as @@ -1373,14 +1375,14 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) unsigned long orig_iova = iova; if (unlikely(domain->ops->unmap == NULL || - domain->ops->pgsize_bitmap == 0UL)) + domain->pgsize_bitmap == 0UL)) return -ENODEV; if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) return -EINVAL; /* find out the minimum page size supported */ - min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap); + min_pagesz = 1 << __ffs(domain->pgsize_bitmap); /* * The virtual address, as well as the size of the mapping, must be @@ -1426,10 +1428,10 @@ size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova, unsigned int i, min_pagesz; int ret; - if (unlikely(domain->ops->pgsize_bitmap == 0UL)) + if (unlikely(domain->pgsize_bitmap == 0UL)) return 0; - min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap); + min_pagesz = 1 << __ffs(domain->pgsize_bitmap); for_each_sg(sg, s, nents, i) { phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset; @@ -1510,7 +1512,7 @@ int iommu_domain_get_attr(struct iommu_domain *domain, break; case DOMAIN_ATTR_PAGING: paging = data; - *paging = (domain->ops->pgsize_bitmap != 0UL); + *paging = (domain->pgsize_bitmap != 0UL); break; case DOMAIN_ATTR_WINDOWS: count = data; diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 929a66a81b2b..e6b25276cfec 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -264,7 +264,7 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data) } /* Update our support page sizes bitmap */ - mtk_iommu_ops.pgsize_bitmap = dom->cfg.pgsize_bitmap; + dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap; writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0], data->base + REG_MMU_PT_BASE_ADDR); diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 75b24e93cedb..15a65823aad9 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -407,7 +407,7 @@ static unsigned long vfio_pgsize_bitmap(struct vfio_iommu *iommu) mutex_lock(&iommu->lock); list_for_each_entry(domain, &iommu->domain_list, next) - bitmap &= domain->domain->ops->pgsize_bitmap; + bitmap &= domain->domain->pgsize_bitmap; mutex_unlock(&iommu->lock); /* diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 8a2570443b80..7811294bc0f7 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -78,6 +78,7 @@ struct iommu_domain_geometry { struct iommu_domain { unsigned type; const struct iommu_ops *ops; + unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */ iommu_fault_handler_t handler; void *handler_token; struct iommu_domain_geometry geometry; @@ -155,7 +156,7 @@ struct iommu_dm_region { * @domain_set_windows: Set the number of windows for a domain * @domain_get_windows: Return the number of windows for a domain * @of_xlate: add OF master IDs to iommu grouping - * @pgsize_bitmap: bitmap of supported page sizes + * @pgsize_bitmap: bitmap of all possible supported page sizes */ struct iommu_ops { bool (*capable)(enum iommu_cap); -- cgit v1.2.3 From 3b6b7e19e31a816ee02a8d4372cbea9ad7db3784 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 13 Apr 2016 17:29:10 +0100 Subject: iommu/dma: Finish optimising higher-order allocations Now that we know exactly which page sizes our caller wants to use in the given domain, we can restrict higher-order allocation attempts to just those sizes, if any, and avoid wasting any time or effort on other sizes which offer no benefit. In the same vein, this also lets us accommodate a minimum order greater than 0 for special cases. Signed-off-by: Robin Murphy Acked-by: Will Deacon Tested-by: Yong Wu Signed-off-by: Joerg Roedel --- arch/arm64/mm/dma-mapping.c | 4 +-- drivers/iommu/dma-iommu.c | 60 +++++++++++++++++++++++++++++---------------- include/linux/dma-iommu.h | 4 +-- 3 files changed, 43 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 5d36907f9b12..41d19a0fc9c0 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -562,8 +562,8 @@ static void *__iommu_alloc_attrs(struct device *dev, size_t size, struct page **pages; pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, coherent); - pages = iommu_dma_alloc(dev, iosize, gfp, ioprot, handle, - flush_page); + pages = iommu_dma_alloc(dev, iosize, gfp, attrs, ioprot, + handle, flush_page); if (!pages) return NULL; diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 99432999b52f..ea5a9ebf0f78 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -190,11 +190,15 @@ static void __iommu_dma_free_pages(struct page **pages, int count) kvfree(pages); } -static struct page **__iommu_dma_alloc_pages(unsigned int count, gfp_t gfp) +static struct page **__iommu_dma_alloc_pages(unsigned int count, + unsigned long order_mask, gfp_t gfp) { struct page **pages; unsigned int i = 0, array_size = count * sizeof(*pages); - unsigned int order = MAX_ORDER; + + order_mask &= (2U << MAX_ORDER) - 1; + if (!order_mask) + return NULL; if (array_size <= PAGE_SIZE) pages = kzalloc(array_size, GFP_KERNEL); @@ -208,36 +212,38 @@ static struct page **__iommu_dma_alloc_pages(unsigned int count, gfp_t gfp) while (count) { struct page *page = NULL; - int j; + unsigned int order_size; /* * Higher-order allocations are a convenience rather * than a necessity, hence using __GFP_NORETRY until - * falling back to single-page allocations. + * falling back to minimum-order allocations. */ - for (order = min_t(unsigned int, order, __fls(count)); - order > 0; order--) { - page = alloc_pages(gfp | __GFP_NORETRY, order); + for (order_mask &= (2U << __fls(count)) - 1; + order_mask; order_mask &= ~order_size) { + unsigned int order = __fls(order_mask); + + order_size = 1U << order; + page = alloc_pages((order_mask - order_size) ? + gfp | __GFP_NORETRY : gfp, order); if (!page) continue; - if (PageCompound(page)) { - if (!split_huge_page(page)) - break; - __free_pages(page, order); - } else { + if (!order) + break; + if (!PageCompound(page)) { split_page(page, order); break; + } else if (!split_huge_page(page)) { + break; } + __free_pages(page, order); } - if (!page) - page = alloc_page(gfp); if (!page) { __iommu_dma_free_pages(pages, i); return NULL; } - j = 1 << order; - count -= j; - while (j--) + count -= order_size; + while (order_size--) pages[i++] = page++; } return pages; @@ -267,6 +273,7 @@ void iommu_dma_free(struct device *dev, struct page **pages, size_t size, * attached to an iommu_dma_domain * @size: Size of buffer in bytes * @gfp: Allocation flags + * @attrs: DMA attributes for this allocation * @prot: IOMMU mapping flags * @handle: Out argument for allocated DMA handle * @flush_page: Arch callback which must ensure PAGE_SIZE bytes from the @@ -278,8 +285,8 @@ void iommu_dma_free(struct device *dev, struct page **pages, size_t size, * Return: Array of struct page pointers describing the buffer, * or NULL on failure. */ -struct page **iommu_dma_alloc(struct device *dev, size_t size, - gfp_t gfp, int prot, dma_addr_t *handle, +struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, + struct dma_attrs *attrs, int prot, dma_addr_t *handle, void (*flush_page)(struct device *, const void *, phys_addr_t)) { struct iommu_domain *domain = iommu_get_domain_for_dev(dev); @@ -288,11 +295,22 @@ struct page **iommu_dma_alloc(struct device *dev, size_t size, struct page **pages; struct sg_table sgt; dma_addr_t dma_addr; - unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; + unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap; *handle = DMA_ERROR_CODE; - pages = __iommu_dma_alloc_pages(count, gfp); + min_size = alloc_sizes & -alloc_sizes; + if (min_size < PAGE_SIZE) { + min_size = PAGE_SIZE; + alloc_sizes |= PAGE_SIZE; + } else { + size = ALIGN(size, min_size); + } + if (dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) + alloc_sizes = min_size; + + count = PAGE_ALIGN(size) >> PAGE_SHIFT; + pages = __iommu_dma_alloc_pages(count, alloc_sizes >> PAGE_SHIFT, gfp); if (!pages) return NULL; diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h index fc481037478a..8443bbb5c071 100644 --- a/include/linux/dma-iommu.h +++ b/include/linux/dma-iommu.h @@ -38,8 +38,8 @@ int dma_direction_to_prot(enum dma_data_direction dir, bool coherent); * These implement the bulk of the relevant DMA mapping callbacks, but require * the arch code to take care of attributes and cache maintenance */ -struct page **iommu_dma_alloc(struct device *dev, size_t size, - gfp_t gfp, int prot, dma_addr_t *handle, +struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, + struct dma_attrs *attrs, int prot, dma_addr_t *handle, void (*flush_page)(struct device *, const void *, phys_addr_t)); void iommu_dma_free(struct device *dev, struct page **pages, size_t size, dma_addr_t *handle); -- cgit v1.2.3