diff options
Diffstat (limited to 'drivers/xen/gntdev.c')
| -rw-r--r-- | drivers/xen/gntdev.c | 220 |
1 files changed, 149 insertions, 71 deletions
diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index bd56653b9bbc..c866a62f766d 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c @@ -6,6 +6,7 @@ * * Copyright (c) 2006-2007, D G Murray. * (c) 2009 Gerd Hoffmann <kraxel@redhat.com> + * (c) 2018 Oleksandr Andrushchenko, EPAM Systems Inc. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -26,10 +27,6 @@ #include <linux/init.h> #include <linux/miscdevice.h> #include <linux/fs.h> -#include <linux/mm.h> -#include <linux/mman.h> -#include <linux/mmu_notifier.h> -#include <linux/types.h> #include <linux/uaccess.h> #include <linux/sched.h> #include <linux/sched/mm.h> @@ -37,6 +34,9 @@ #include <linux/slab.h> #include <linux/highmem.h> #include <linux/refcount.h> +#ifdef CONFIG_XEN_GRANT_DMA_ALLOC +#include <linux/of_device.h> +#endif #include <xen/xen.h> #include <xen/grant_table.h> @@ -47,6 +47,11 @@ #include <asm/xen/hypervisor.h> #include <asm/xen/hypercall.h> +#include "gntdev-common.h" +#ifdef CONFIG_XEN_GNTDEV_DMABUF +#include "gntdev-dmabuf.h" +#endif + MODULE_LICENSE("GPL"); MODULE_AUTHOR("Derek G. Murray <Derek.Murray@cl.cam.ac.uk>, " "Gerd Hoffmann <kraxel@redhat.com>"); @@ -62,51 +67,23 @@ static atomic_t pages_mapped = ATOMIC_INIT(0); static int use_ptemod; #define populate_freeable_maps use_ptemod -struct gntdev_priv { - /* maps with visible offsets in the file descriptor */ - struct list_head maps; - /* maps that are not visible; will be freed on munmap. - * Only populated if populate_freeable_maps == 1 */ - struct list_head freeable_maps; - /* lock protects maps and freeable_maps */ - struct mutex lock; - struct mm_struct *mm; - struct mmu_notifier mn; -}; - -struct unmap_notify { - int flags; - /* Address relative to the start of the grant_map */ - int addr; - int event; -}; - -struct grant_map { - struct list_head next; - struct vm_area_struct *vma; - int index; - int count; - int flags; - refcount_t users; - struct unmap_notify notify; - struct ioctl_gntdev_grant_ref *grants; - struct gnttab_map_grant_ref *map_ops; - struct gnttab_unmap_grant_ref *unmap_ops; - struct gnttab_map_grant_ref *kmap_ops; - struct gnttab_unmap_grant_ref *kunmap_ops; - struct page **pages; - unsigned long pages_vm_start; -}; +static int unmap_grant_pages(struct gntdev_grant_map *map, + int offset, int pages); -static int unmap_grant_pages(struct grant_map *map, int offset, int pages); +static struct miscdevice gntdev_miscdev; /* ------------------------------------------------------------------ */ +bool gntdev_account_mapped_pages(int count) +{ + return atomic_add_return(count, &pages_mapped) > limit; +} + static void gntdev_print_maps(struct gntdev_priv *priv, char *text, int text_index) { #ifdef DEBUG - struct grant_map *map; + struct gntdev_grant_map *map; pr_debug("%s: maps list (priv %p)\n", __func__, priv); list_for_each_entry(map, &priv->maps, next) @@ -116,13 +93,32 @@ static void gntdev_print_maps(struct gntdev_priv *priv, #endif } -static void gntdev_free_map(struct grant_map *map) +static void gntdev_free_map(struct gntdev_grant_map *map) { if (map == NULL) return; +#ifdef CONFIG_XEN_GRANT_DMA_ALLOC + if (map->dma_vaddr) { + struct gnttab_dma_alloc_args args; + + args.dev = map->dma_dev; + args.coherent = !!(map->dma_flags & GNTDEV_DMA_FLAG_COHERENT); + args.nr_pages = map->count; + args.pages = map->pages; + args.frames = map->frames; + args.vaddr = map->dma_vaddr; + args.dev_bus_addr = map->dma_bus_addr; + + gnttab_dma_free_pages(&args); + } else +#endif if (map->pages) gnttab_free_pages(map->count, map->pages); + +#ifdef CONFIG_XEN_GRANT_DMA_ALLOC + kfree(map->frames); +#endif kfree(map->pages); kfree(map->grants); kfree(map->map_ops); @@ -132,12 +128,13 @@ static void gntdev_free_map(struct grant_map *map) kfree(map); } -static struct grant_map *gntdev_alloc_map(struct gntdev_priv *priv, int count) +struct gntdev_grant_map *gntdev_alloc_map(struct gntdev_priv *priv, int count, + int dma_flags) { - struct grant_map *add; + struct gntdev_grant_map *add; int i; - add = kzalloc(sizeof(struct grant_map), GFP_KERNEL); + add = kzalloc(sizeof(*add), GFP_KERNEL); if (NULL == add) return NULL; @@ -155,6 +152,37 @@ static struct grant_map *gntdev_alloc_map(struct gntdev_priv *priv, int count) NULL == add->pages) goto err; +#ifdef CONFIG_XEN_GRANT_DMA_ALLOC + add->dma_flags = dma_flags; + + /* + * Check if this mapping is requested to be backed + * by a DMA buffer. + */ + if (dma_flags & (GNTDEV_DMA_FLAG_WC | GNTDEV_DMA_FLAG_COHERENT)) { + struct gnttab_dma_alloc_args args; + + add->frames = kcalloc(count, sizeof(add->frames[0]), + GFP_KERNEL); + if (!add->frames) + goto err; + + /* Remember the device, so we can free DMA memory. */ + add->dma_dev = priv->dma_dev; + + args.dev = priv->dma_dev; + args.coherent = !!(dma_flags & GNTDEV_DMA_FLAG_COHERENT); + args.nr_pages = count; + args.pages = add->pages; + args.frames = add->frames; + + if (gnttab_dma_alloc_pages(&args)) + goto err; + + add->dma_vaddr = args.vaddr; + add->dma_bus_addr = args.dev_bus_addr; + } else +#endif if (gnttab_alloc_pages(count, add->pages)) goto err; @@ -176,9 +204,9 @@ err: return NULL; } -static void gntdev_add_map(struct gntdev_priv *priv, struct grant_map *add) +void gntdev_add_map(struct gntdev_priv *priv, struct gntdev_grant_map *add) { - struct grant_map *map; + struct gntdev_grant_map *map; list_for_each_entry(map, &priv->maps, next) { if (add->index + add->count < map->index) { @@ -193,10 +221,10 @@ done: gntdev_print_maps(priv, "[new]", add->index); } -static struct grant_map *gntdev_find_map_index(struct gntdev_priv *priv, - int index, int count) +static struct gntdev_grant_map *gntdev_find_map_index(struct gntdev_priv *priv, + int index, int count) { - struct grant_map *map; + struct gntdev_grant_map *map; list_for_each_entry(map, &priv->maps, next) { if (map->index != index) @@ -208,7 +236,7 @@ static struct grant_map *gntdev_find_map_index(struct gntdev_priv *priv, return NULL; } -static void gntdev_put_map(struct gntdev_priv *priv, struct grant_map *map) +void gntdev_put_map(struct gntdev_priv *priv, struct gntdev_grant_map *map) { if (!map) return; @@ -239,7 +267,7 @@ static void gntdev_put_map(struct gntdev_priv *priv, struct grant_map *map) static int find_grant_ptes(pte_t *pte, pgtable_t token, unsigned long addr, void *data) { - struct grant_map *map = data; + struct gntdev_grant_map *map = data; unsigned int pgnr = (addr - map->vma->vm_start) >> PAGE_SHIFT; int flags = map->flags | GNTMAP_application_map | GNTMAP_contains_pte; u64 pte_maddr; @@ -272,7 +300,7 @@ static int set_grant_ptes_as_special(pte_t *pte, pgtable_t token, } #endif -static int map_grant_pages(struct grant_map *map) +int gntdev_map_grant_pages(struct gntdev_grant_map *map) { int i, err = 0; @@ -325,11 +353,20 @@ static int map_grant_pages(struct grant_map *map) map->unmap_ops[i].handle = map->map_ops[i].handle; if (use_ptemod) map->kunmap_ops[i].handle = map->kmap_ops[i].handle; +#ifdef CONFIG_XEN_GRANT_DMA_ALLOC + else if (map->dma_vaddr) { + unsigned long bfn; + + bfn = pfn_to_bfn(page_to_pfn(map->pages[i])); + map->unmap_ops[i].dev_bus_addr = __pfn_to_phys(bfn); + } +#endif } return err; } -static int __unmap_grant_pages(struct grant_map *map, int offset, int pages) +static int __unmap_grant_pages(struct gntdev_grant_map *map, int offset, + int pages) { int i, err = 0; struct gntab_unmap_queue_data unmap_data; @@ -364,7 +401,8 @@ static int __unmap_grant_pages(struct grant_map *map, int offset, int pages) return err; } -static int unmap_grant_pages(struct grant_map *map, int offset, int pages) +static int unmap_grant_pages(struct gntdev_grant_map *map, int offset, + int pages) { int range, err = 0; @@ -396,7 +434,7 @@ static int unmap_grant_pages(struct grant_map *map, int offset, int pages) static void gntdev_vma_open(struct vm_area_struct *vma) { - struct grant_map *map = vma->vm_private_data; + struct gntdev_grant_map *map = vma->vm_private_data; pr_debug("gntdev_vma_open %p\n", vma); refcount_inc(&map->users); @@ -404,7 +442,7 @@ static void gntdev_vma_open(struct vm_area_struct *vma) static void gntdev_vma_close(struct vm_area_struct *vma) { - struct grant_map *map = vma->vm_private_data; + struct gntdev_grant_map *map = vma->vm_private_data; struct file *file = vma->vm_file; struct gntdev_priv *priv = file->private_data; @@ -428,7 +466,7 @@ static void gntdev_vma_close(struct vm_area_struct *vma) static struct page *gntdev_vma_find_special_page(struct vm_area_struct *vma, unsigned long addr) { - struct grant_map *map = vma->vm_private_data; + struct gntdev_grant_map *map = vma->vm_private_data; return map->pages[(addr - map->pages_vm_start) >> PAGE_SHIFT]; } @@ -441,7 +479,7 @@ static const struct vm_operations_struct gntdev_vmops = { /* ------------------------------------------------------------------ */ -static void unmap_if_in_range(struct grant_map *map, +static void unmap_if_in_range(struct gntdev_grant_map *map, unsigned long start, unsigned long end) { unsigned long mstart, mend; @@ -470,7 +508,7 @@ static void mn_invl_range_start(struct mmu_notifier *mn, unsigned long start, unsigned long end) { struct gntdev_priv *priv = container_of(mn, struct gntdev_priv, mn); - struct grant_map *map; + struct gntdev_grant_map *map; mutex_lock(&priv->lock); list_for_each_entry(map, &priv->maps, next) { @@ -486,7 +524,7 @@ static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm) { struct gntdev_priv *priv = container_of(mn, struct gntdev_priv, mn); - struct grant_map *map; + struct gntdev_grant_map *map; int err; mutex_lock(&priv->lock); @@ -531,6 +569,15 @@ static int gntdev_open(struct inode *inode, struct file *flip) INIT_LIST_HEAD(&priv->freeable_maps); mutex_init(&priv->lock); +#ifdef CONFIG_XEN_GNTDEV_DMABUF + priv->dmabuf_priv = gntdev_dmabuf_init(); + if (IS_ERR(priv->dmabuf_priv)) { + ret = PTR_ERR(priv->dmabuf_priv); + kfree(priv); + return ret; + } +#endif + if (use_ptemod) { priv->mm = get_task_mm(current); if (!priv->mm) { @@ -548,6 +595,17 @@ static int gntdev_open(struct inode *inode, struct file *flip) } flip->private_data = priv; +#ifdef CONFIG_XEN_GRANT_DMA_ALLOC + priv->dma_dev = gntdev_miscdev.this_device; + + /* + * The device is not spawn from a device tree, so arch_setup_dma_ops + * is not called, thus leaving the device with dummy DMA ops. + * Fix this by calling of_dma_configure() with a NULL node to set + * default DMA ops. + */ + of_dma_configure(priv->dma_dev, NULL, true); +#endif pr_debug("priv %p\n", priv); return 0; @@ -556,21 +614,27 @@ static int gntdev_open(struct inode *inode, struct file *flip) static int gntdev_release(struct inode *inode, struct file *flip) { struct gntdev_priv *priv = flip->private_data; - struct grant_map *map; + struct gntdev_grant_map *map; pr_debug("priv %p\n", priv); mutex_lock(&priv->lock); while (!list_empty(&priv->maps)) { - map = list_entry(priv->maps.next, struct grant_map, next); + map = list_entry(priv->maps.next, + struct gntdev_grant_map, next); list_del(&map->next); gntdev_put_map(NULL /* already removed */, map); } WARN_ON(!list_empty(&priv->freeable_maps)); mutex_unlock(&priv->lock); +#ifdef CONFIG_XEN_GNTDEV_DMABUF + gntdev_dmabuf_fini(priv->dmabuf_priv); +#endif + if (use_ptemod) mmu_notifier_unregister(&priv->mn, priv->mm); + kfree(priv); return 0; } @@ -579,7 +643,7 @@ static long gntdev_ioctl_map_grant_ref(struct gntdev_priv *priv, struct ioctl_gntdev_map_grant_ref __user *u) { struct ioctl_gntdev_map_grant_ref op; - struct grant_map *map; + struct gntdev_grant_map *map; int err; if (copy_from_user(&op, u, sizeof(op)) != 0) @@ -589,11 +653,11 @@ static long gntdev_ioctl_map_grant_ref(struct gntdev_priv *priv, return -EINVAL; err = -ENOMEM; - map = gntdev_alloc_map(priv, op.count); + map = gntdev_alloc_map(priv, op.count, 0 /* This is not a dma-buf. */); if (!map) return err; - if (unlikely(atomic_add_return(op.count, &pages_mapped) > limit)) { + if (unlikely(gntdev_account_mapped_pages(op.count))) { pr_debug("can't map: over limit\n"); gntdev_put_map(NULL, map); return err; @@ -620,7 +684,7 @@ static long gntdev_ioctl_unmap_grant_ref(struct gntdev_priv *priv, struct ioctl_gntdev_unmap_grant_ref __user *u) { struct ioctl_gntdev_unmap_grant_ref op; - struct grant_map *map; + struct gntdev_grant_map *map; int err = -ENOENT; if (copy_from_user(&op, u, sizeof(op)) != 0) @@ -646,7 +710,7 @@ static long gntdev_ioctl_get_offset_for_vaddr(struct gntdev_priv *priv, { struct ioctl_gntdev_get_offset_for_vaddr op; struct vm_area_struct *vma; - struct grant_map *map; + struct gntdev_grant_map *map; int rv = -EINVAL; if (copy_from_user(&op, u, sizeof(op)) != 0) @@ -677,7 +741,7 @@ static long gntdev_ioctl_get_offset_for_vaddr(struct gntdev_priv *priv, static long gntdev_ioctl_notify(struct gntdev_priv *priv, void __user *u) { struct ioctl_gntdev_unmap_notify op; - struct grant_map *map; + struct gntdev_grant_map *map; int rc; int out_flags; unsigned int out_event; @@ -962,6 +1026,20 @@ static long gntdev_ioctl(struct file *flip, case IOCTL_GNTDEV_GRANT_COPY: return gntdev_ioctl_grant_copy(priv, ptr); +#ifdef CONFIG_XEN_GNTDEV_DMABUF + case IOCTL_GNTDEV_DMABUF_EXP_FROM_REFS: + return gntdev_ioctl_dmabuf_exp_from_refs(priv, use_ptemod, ptr); + + case IOCTL_GNTDEV_DMABUF_EXP_WAIT_RELEASED: + return gntdev_ioctl_dmabuf_exp_wait_released(priv, ptr); + + case IOCTL_GNTDEV_DMABUF_IMP_TO_REFS: + return gntdev_ioctl_dmabuf_imp_to_refs(priv, ptr); + + case IOCTL_GNTDEV_DMABUF_IMP_RELEASE: + return gntdev_ioctl_dmabuf_imp_release(priv, ptr); +#endif + default: pr_debug("priv %p, unknown cmd %x\n", priv, cmd); return -ENOIOCTLCMD; @@ -975,7 +1053,7 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma) struct gntdev_priv *priv = flip->private_data; int index = vma->vm_pgoff; int count = vma_pages(vma); - struct grant_map *map; + struct gntdev_grant_map *map; int i, err = -EINVAL; if ((vma->vm_flags & VM_WRITE) && !(vma->vm_flags & VM_SHARED)) @@ -1032,7 +1110,7 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma) } } - err = map_grant_pages(map); + err = gntdev_map_grant_pages(map); if (err) goto out_put_map; |