diff options
Diffstat (limited to 'arch/arm')
| -rw-r--r-- | arch/arm/include/asm/device.h | 1 | ||||
| -rw-r--r-- | arch/arm/include/asm/dma-mapping.h | 7 | ||||
| -rw-r--r-- | arch/arm/include/asm/xen/page-coherent.h | 66 | ||||
| -rw-r--r-- | arch/arm/include/asm/xen/page.h | 4 | ||||
| -rw-r--r-- | arch/arm/xen/Makefile | 2 | ||||
| -rw-r--r-- | arch/arm/xen/enlighten.c | 5 | ||||
| -rw-r--r-- | arch/arm/xen/mm.c | 121 | ||||
| -rw-r--r-- | arch/arm/xen/mm32.c | 202 |
8 files changed, 191 insertions, 217 deletions
diff --git a/arch/arm/include/asm/device.h b/arch/arm/include/asm/device.h index dc662fca9230..4111592f0130 100644 --- a/arch/arm/include/asm/device.h +++ b/arch/arm/include/asm/device.h @@ -17,6 +17,7 @@ struct dev_archdata { #ifdef CONFIG_ARM_DMA_USE_IOMMU struct dma_iommu_mapping *mapping; #endif + bool dma_coherent; }; struct omap_device; diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index 85738b200023..e6e3446abdf6 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -123,11 +123,18 @@ static inline unsigned long dma_max_pfn(struct device *dev) static inline int set_arch_dma_coherent_ops(struct device *dev) { + dev->archdata.dma_coherent = true; set_dma_ops(dev, &arm_coherent_dma_ops); return 0; } #define set_arch_dma_coherent_ops(dev) set_arch_dma_coherent_ops(dev) +/* do not use this function in a driver */ +static inline bool is_device_dma_coherent(struct device *dev) +{ + return dev->archdata.dma_coherent; +} + static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) { unsigned int offset = paddr & ~PAGE_MASK; diff --git a/arch/arm/include/asm/xen/page-coherent.h b/arch/arm/include/asm/xen/page-coherent.h index e8275ea88e88..efd562412850 100644 --- a/arch/arm/include/asm/xen/page-coherent.h +++ b/arch/arm/include/asm/xen/page-coherent.h @@ -5,6 +5,18 @@ #include <linux/dma-attrs.h> #include <linux/dma-mapping.h> +void __xen_dma_map_page(struct device *hwdev, struct page *page, + dma_addr_t dev_addr, unsigned long offset, size_t size, + enum dma_data_direction dir, struct dma_attrs *attrs); +void __xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir, + struct dma_attrs *attrs); +void __xen_dma_sync_single_for_cpu(struct device *hwdev, + dma_addr_t handle, size_t size, enum dma_data_direction dir); + +void __xen_dma_sync_single_for_device(struct device *hwdev, + dma_addr_t handle, size_t size, enum dma_data_direction dir); + static inline void *xen_alloc_coherent_pages(struct device *hwdev, size_t size, dma_addr_t *dma_handle, gfp_t flags, struct dma_attrs *attrs) @@ -20,20 +32,56 @@ static inline void xen_free_coherent_pages(struct device *hwdev, size_t size, } static inline void xen_dma_map_page(struct device *hwdev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + dma_addr_t dev_addr, unsigned long offset, size_t size, + enum dma_data_direction dir, struct dma_attrs *attrs) { - __generic_dma_ops(hwdev)->map_page(hwdev, page, offset, size, dir, attrs); + bool local = PFN_DOWN(dev_addr) == page_to_pfn(page); + /* Dom0 is mapped 1:1, so if pfn == mfn the page is local otherwise + * is a foreign page grant-mapped in dom0. If the page is local we + * can safely call the native dma_ops function, otherwise we call + * the xen specific function. */ + if (local) + __generic_dma_ops(hwdev)->map_page(hwdev, page, offset, size, dir, attrs); + else + __xen_dma_map_page(hwdev, page, dev_addr, offset, size, dir, attrs); } -void xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle, +static inline void xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs); + struct dma_attrs *attrs) +{ + unsigned long pfn = PFN_DOWN(handle); + /* Dom0 is mapped 1:1, so calling pfn_valid on a foreign mfn will + * always return false. If the page is local we can safely call the + * native dma_ops function, otherwise we call the xen specific + * function. */ + if (pfn_valid(pfn)) { + if (__generic_dma_ops(hwdev)->unmap_page) + __generic_dma_ops(hwdev)->unmap_page(hwdev, handle, size, dir, attrs); + } else + __xen_dma_unmap_page(hwdev, handle, size, dir, attrs); +} -void xen_dma_sync_single_for_cpu(struct device *hwdev, - dma_addr_t handle, size_t size, enum dma_data_direction dir); +static inline void xen_dma_sync_single_for_cpu(struct device *hwdev, + dma_addr_t handle, size_t size, enum dma_data_direction dir) +{ + unsigned long pfn = PFN_DOWN(handle); + if (pfn_valid(pfn)) { + if (__generic_dma_ops(hwdev)->sync_single_for_cpu) + __generic_dma_ops(hwdev)->sync_single_for_cpu(hwdev, handle, size, dir); + } else + __xen_dma_sync_single_for_cpu(hwdev, handle, size, dir); +} -void xen_dma_sync_single_for_device(struct device *hwdev, - dma_addr_t handle, size_t size, enum dma_data_direction dir); +static inline void xen_dma_sync_single_for_device(struct device *hwdev, + dma_addr_t handle, size_t size, enum dma_data_direction dir) +{ + unsigned long pfn = PFN_DOWN(handle); + if (pfn_valid(pfn)) { + if (__generic_dma_ops(hwdev)->sync_single_for_device) + __generic_dma_ops(hwdev)->sync_single_for_device(hwdev, handle, size, dir); + } else + __xen_dma_sync_single_for_device(hwdev, handle, size, dir); +} #endif /* _ASM_ARM_XEN_PAGE_COHERENT_H */ diff --git a/arch/arm/include/asm/xen/page.h b/arch/arm/include/asm/xen/page.h index 135c24a5ba26..68c739b3fdf4 100644 --- a/arch/arm/include/asm/xen/page.h +++ b/arch/arm/include/asm/xen/page.h @@ -107,4 +107,8 @@ static inline bool set_phys_to_machine(unsigned long pfn, unsigned long mfn) #define xen_remap(cookie, size) ioremap_cache((cookie), (size)) #define xen_unmap(cookie) iounmap((cookie)) +bool xen_arch_need_swiotlb(struct device *dev, + unsigned long pfn, + unsigned long mfn); + #endif /* _ASM_ARM_XEN_PAGE_H */ diff --git a/arch/arm/xen/Makefile b/arch/arm/xen/Makefile index 1f85bfe6b470..12969523414c 100644 --- a/arch/arm/xen/Makefile +++ b/arch/arm/xen/Makefile @@ -1 +1 @@ -obj-y := enlighten.o hypercall.o grant-table.o p2m.o mm.o mm32.o +obj-y := enlighten.o hypercall.o grant-table.o p2m.o mm.o diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c index 0e15f011f9c8..c7ca936ebd99 100644 --- a/arch/arm/xen/enlighten.c +++ b/arch/arm/xen/enlighten.c @@ -261,11 +261,6 @@ static int __init xen_guest_init(void) xen_setup_features(); - if (!xen_feature(XENFEAT_grant_map_identity)) { - pr_warn("Please upgrade your Xen.\n" - "If your platform has any non-coherent DMA devices, they won't work properly.\n"); - } - if (xen_feature(XENFEAT_dom0)) xen_start_info->flags |= SIF_INITDOMAIN|SIF_PRIVILEGED; else diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c index b0e77de99148..351b24a979d4 100644 --- a/arch/arm/xen/mm.c +++ b/arch/arm/xen/mm.c @@ -1,6 +1,10 @@ +#include <linux/cpu.h> +#include <linux/dma-mapping.h> #include <linux/bootmem.h> #include <linux/gfp.h> +#include <linux/highmem.h> #include <linux/export.h> +#include <linux/of_address.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/dma-mapping.h> @@ -8,6 +12,7 @@ #include <linux/swiotlb.h> #include <xen/xen.h> +#include <xen/interface/grant_table.h> #include <xen/interface/memory.h> #include <xen/swiotlb-xen.h> @@ -16,6 +21,114 @@ #include <asm/xen/hypercall.h> #include <asm/xen/interface.h> +enum dma_cache_op { + DMA_UNMAP, + DMA_MAP, +}; +static bool hypercall_cflush = false; + +/* functions called by SWIOTLB */ + +static void dma_cache_maint(dma_addr_t handle, unsigned long offset, + size_t size, enum dma_data_direction dir, enum dma_cache_op op) +{ + struct gnttab_cache_flush cflush; + unsigned long pfn; + size_t left = size; + + pfn = (handle >> PAGE_SHIFT) + offset / PAGE_SIZE; + offset %= PAGE_SIZE; + + do { + size_t len = left; + + /* buffers in highmem or foreign pages cannot cross page + * boundaries */ + if (len + offset > PAGE_SIZE) + len = PAGE_SIZE - offset; + + cflush.op = 0; + cflush.a.dev_bus_addr = pfn << PAGE_SHIFT; + cflush.offset = offset; + cflush.length = len; + + if (op == DMA_UNMAP && dir != DMA_TO_DEVICE) + cflush.op = GNTTAB_CACHE_INVAL; + if (op == DMA_MAP) { + if (dir == DMA_FROM_DEVICE) + cflush.op = GNTTAB_CACHE_INVAL; + else + cflush.op = GNTTAB_CACHE_CLEAN; + } + if (cflush.op) + HYPERVISOR_grant_table_op(GNTTABOP_cache_flush, &cflush, 1); + + offset = 0; + pfn++; + left -= len; + } while (left); +} + +static void __xen_dma_page_dev_to_cpu(struct device *hwdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir) +{ + dma_cache_maint(handle & PAGE_MASK, handle & ~PAGE_MASK, size, dir, DMA_UNMAP); +} + +static void __xen_dma_page_cpu_to_dev(struct device *hwdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir) +{ + dma_cache_maint(handle & PAGE_MASK, handle & ~PAGE_MASK, size, dir, DMA_MAP); +} + +void __xen_dma_map_page(struct device *hwdev, struct page *page, + dma_addr_t dev_addr, unsigned long offset, size_t size, + enum dma_data_direction dir, struct dma_attrs *attrs) +{ + if (is_device_dma_coherent(hwdev)) + return; + if (dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) + return; + + __xen_dma_page_cpu_to_dev(hwdev, dev_addr, size, dir); +} + +void __xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir, + struct dma_attrs *attrs) + +{ + if (is_device_dma_coherent(hwdev)) + return; + if (dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) + return; + + __xen_dma_page_dev_to_cpu(hwdev, handle, size, dir); +} + +void __xen_dma_sync_single_for_cpu(struct device *hwdev, + dma_addr_t handle, size_t size, enum dma_data_direction dir) +{ + if (is_device_dma_coherent(hwdev)) + return; + __xen_dma_page_dev_to_cpu(hwdev, handle, size, dir); +} + +void __xen_dma_sync_single_for_device(struct device *hwdev, + dma_addr_t handle, size_t size, enum dma_data_direction dir) +{ + if (is_device_dma_coherent(hwdev)) + return; + __xen_dma_page_cpu_to_dev(hwdev, handle, size, dir); +} + +bool xen_arch_need_swiotlb(struct device *dev, + unsigned long pfn, + unsigned long mfn) +{ + return (!hypercall_cflush && (pfn != mfn) && !is_device_dma_coherent(dev)); +} + int xen_create_contiguous_region(phys_addr_t pstart, unsigned int order, unsigned int address_bits, dma_addr_t *dma_handle) @@ -56,10 +169,18 @@ static struct dma_map_ops xen_swiotlb_dma_ops = { int __init xen_mm_init(void) { + struct gnttab_cache_flush cflush; if (!xen_initial_domain()) return 0; xen_swiotlb_init(1, false); xen_dma_ops = &xen_swiotlb_dma_ops; + + cflush.op = 0; + cflush.a.dev_bus_addr = 0; + cflush.offset = 0; + cflush.length = 0; + if (HYPERVISOR_grant_table_op(GNTTABOP_cache_flush, &cflush, 1) != -ENOSYS) + hypercall_cflush = true; return 0; } arch_initcall(xen_mm_init); diff --git a/arch/arm/xen/mm32.c b/arch/arm/xen/mm32.c deleted file mode 100644 index 3b99860fd7ae..000000000000 --- a/arch/arm/xen/mm32.c +++ /dev/null @@ -1,202 +0,0 @@ -#include <linux/cpu.h> -#include <linux/dma-mapping.h> -#include <linux/gfp.h> -#include <linux/highmem.h> - -#include <xen/features.h> - -static DEFINE_PER_CPU(unsigned long, xen_mm32_scratch_virt); -static DEFINE_PER_CPU(pte_t *, xen_mm32_scratch_ptep); - -static int alloc_xen_mm32_scratch_page(int cpu) -{ - struct page *page; - unsigned long virt; - pmd_t *pmdp; - pte_t *ptep; - - if (per_cpu(xen_mm32_scratch_ptep, cpu) != NULL) - return 0; - - page = alloc_page(GFP_KERNEL); - if (page == NULL) { - pr_warn("Failed to allocate xen_mm32_scratch_page for cpu %d\n", cpu); - return -ENOMEM; - } - - virt = (unsigned long)__va(page_to_phys(page)); - pmdp = pmd_offset(pud_offset(pgd_offset_k(virt), virt), virt); - ptep = pte_offset_kernel(pmdp, virt); - - per_cpu(xen_mm32_scratch_virt, cpu) = virt; - per_cpu(xen_mm32_scratch_ptep, cpu) = ptep; - - return 0; -} - -static int xen_mm32_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) -{ - int cpu = (long)hcpu; - switch (action) { - case CPU_UP_PREPARE: - if (alloc_xen_mm32_scratch_page(cpu)) - return NOTIFY_BAD; - break; - default: - break; - } - return NOTIFY_OK; -} - -static struct notifier_block xen_mm32_cpu_notifier = { - .notifier_call = xen_mm32_cpu_notify, -}; - -static void* xen_mm32_remap_page(dma_addr_t handle) -{ - unsigned long virt = get_cpu_var(xen_mm32_scratch_virt); - pte_t *ptep = __get_cpu_var(xen_mm32_scratch_ptep); - - *ptep = pfn_pte(handle >> PAGE_SHIFT, PAGE_KERNEL); - local_flush_tlb_kernel_page(virt); - - return (void*)virt; -} - -static void xen_mm32_unmap(void *vaddr) -{ - put_cpu_var(xen_mm32_scratch_virt); -} - - -/* functions called by SWIOTLB */ - -static void dma_cache_maint(dma_addr_t handle, unsigned long offset, - size_t size, enum dma_data_direction dir, - void (*op)(const void *, size_t, int)) -{ - unsigned long pfn; - size_t left = size; - - pfn = (handle >> PAGE_SHIFT) + offset / PAGE_SIZE; - offset %= PAGE_SIZE; - - do { - size_t len = left; - void *vaddr; - - if (!pfn_valid(pfn)) - { - /* Cannot map the page, we don't know its physical address. - * Return and hope for the best */ - if (!xen_feature(XENFEAT_grant_map_identity)) - return; - vaddr = xen_mm32_remap_page(handle) + offset; - op(vaddr, len, dir); - xen_mm32_unmap(vaddr - offset); - } else { - struct page *page = pfn_to_page(pfn); - - if (PageHighMem(page)) { - if (len + offset > PAGE_SIZE) - len = PAGE_SIZE - offset; - - if (cache_is_vipt_nonaliasing()) { - vaddr = kmap_atomic(page); - op(vaddr + offset, len, dir); - kunmap_atomic(vaddr); - } else { - vaddr = kmap_high_get(page); - if (vaddr) { - op(vaddr + offset, len, dir); - kunmap_high(page); - } - } - } else { - vaddr = page_address(page) + offset; - op(vaddr, len, dir); - } - } - - offset = 0; - pfn++; - left -= len; - } while (left); -} - -static void __xen_dma_page_dev_to_cpu(struct device *hwdev, dma_addr_t handle, - size_t size, enum dma_data_direction dir) -{ - /* Cannot use __dma_page_dev_to_cpu because we don't have a - * struct page for handle */ - - if (dir != DMA_TO_DEVICE) - outer_inv_range(handle, handle + size); - - dma_cache_maint(handle & PAGE_MASK, handle & ~PAGE_MASK, size, dir, dmac_unmap_area); -} - -static void __xen_dma_page_cpu_to_dev(struct device *hwdev, dma_addr_t handle, - size_t size, enum dma_data_direction dir) -{ - - dma_cache_maint(handle & PAGE_MASK, handle & ~PAGE_MASK, size, dir, dmac_map_area); - - if (dir == DMA_FROM_DEVICE) { - outer_inv_range(handle, handle + size); - } else { - outer_clean_range(handle, handle + size); - } -} - -void xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) - -{ - if (!__generic_dma_ops(hwdev)->unmap_page) - return; - if (dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) - return; - - __xen_dma_page_dev_to_cpu(hwdev, handle, size, dir); -} - -void xen_dma_sync_single_for_cpu(struct device *hwdev, - dma_addr_t handle, size_t size, enum dma_data_direction dir) -{ - if (!__generic_dma_ops(hwdev)->sync_single_for_cpu) - return; - __xen_dma_page_dev_to_cpu(hwdev, handle, size, dir); -} - -void xen_dma_sync_single_for_device(struct device *hwdev, - dma_addr_t handle, size_t size, enum dma_data_direction dir) -{ - if (!__generic_dma_ops(hwdev)->sync_single_for_device) - return; - __xen_dma_page_cpu_to_dev(hwdev, handle, size, dir); -} - -int __init xen_mm32_init(void) -{ - int cpu; - - if (!xen_initial_domain()) - return 0; - - register_cpu_notifier(&xen_mm32_cpu_notifier); - get_online_cpus(); - for_each_online_cpu(cpu) { - if (alloc_xen_mm32_scratch_page(cpu)) { - put_online_cpus(); - unregister_cpu_notifier(&xen_mm32_cpu_notifier); - return -ENOMEM; - } - } - put_online_cpus(); - - return 0; -} -arch_initcall(xen_mm32_init); |