From d26c96c8102549f91eb0bea6196d54711ab52176 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Tue, 2 Feb 2016 21:46:35 -0800 Subject: vring: Introduce vring_use_dma_api() This is a kludge, but no one has come up with a a better idea yet. We'll introduce DMA API support guarded by vring_use_dma_api(). Eventually we may be able to return true on more and more systems, and hopefully we can get rid of vring_use_dma_api() entirely some day. Signed-off-by: Andy Lutomirski Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index e12e385f7ac3..ab0be6c084f6 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -104,6 +104,30 @@ struct vring_virtqueue { #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) +/* + * The interaction between virtio and a possible IOMMU is a mess. + * + * On most systems with virtio, physical addresses match bus addresses, + * and it doesn't particularly matter whether we use the DMA API. + * + * On some systems, including Xen and any system with a physical device + * that speaks virtio behind a physical IOMMU, we must use the DMA API + * for virtio DMA to work at all. + * + * On other systems, including SPARC and PPC64, virtio-pci devices are + * enumerated as though they are behind an IOMMU, but the virtio host + * ignores the IOMMU, so we must either pretend that the IOMMU isn't + * there or somehow map everything as the identity. + * + * For the time being, we preserve historic behavior and bypass the DMA + * API. + */ + +static bool vring_use_dma_api(struct virtio_device *vdev) +{ + return false; +} + static struct vring_desc *alloc_indirect(struct virtqueue *_vq, unsigned int total_sg, gfp_t gfp) { -- cgit v1.2.3 From 780bc7903a32edb63be138487fd981694d993610 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Tue, 2 Feb 2016 21:46:36 -0800 Subject: virtio_ring: Support DMA APIs virtio_ring currently sends the device (usually a hypervisor) physical addresses of its I/O buffers. This is okay when DMA addresses and physical addresses are the same thing, but this isn't always the case. For example, this never works on Xen guests, and it is likely to fail if a physical "virtio" device ever ends up behind an IOMMU or swiotlb. The immediate use case for me is to enable virtio on Xen guests. For that to work, we need vring to support DMA address translation as well as a corresponding change to virtio_pci or to another driver. Signed-off-by: Andy Lutomirski Signed-off-by: Michael S. Tsirkin --- drivers/virtio/Kconfig | 2 +- drivers/virtio/virtio_ring.c | 200 ++++++++++++++++++++++++++++++++------- tools/virtio/linux/dma-mapping.h | 17 ++++ 3 files changed, 183 insertions(+), 36 deletions(-) create mode 100644 tools/virtio/linux/dma-mapping.h (limited to 'drivers') diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index cab9f3f63a38..77590320d44c 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -60,7 +60,7 @@ config VIRTIO_INPUT config VIRTIO_MMIO tristate "Platform bus driver for memory mapped virtio devices" - depends on HAS_IOMEM + depends on HAS_IOMEM && HAS_DMA select VIRTIO ---help--- This drivers provides support for memory mapped virtio diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index ab0be6c084f6..9abc008ff7ea 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -24,6 +24,7 @@ #include #include #include +#include #ifdef DEBUG /* For development, we want to crash whenever the ring is screwed. */ @@ -54,6 +55,11 @@ #define END_USE(vq) #endif +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + struct vring_virtqueue { struct virtqueue vq; @@ -98,8 +104,8 @@ struct vring_virtqueue { ktime_t last_add_time; #endif - /* Tokens for callbacks. */ - void *data[]; + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; }; #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) @@ -128,6 +134,79 @@ static bool vring_use_dma_api(struct virtio_device *vdev) return false; } +/* + * The DMA ops on various arches are rather gnarly right now, and + * making all of the arch DMA ops work on the vring device itself + * is a mess. For now, we use the parent device for DMA ops. + */ +struct device *vring_dma_dev(const struct vring_virtqueue *vq) +{ + return vq->vq.vdev->dev.parent; +} + +/* Map one sg entry. */ +static dma_addr_t vring_map_one_sg(const struct vring_virtqueue *vq, + struct scatterlist *sg, + enum dma_data_direction direction) +{ + if (!vring_use_dma_api(vq->vq.vdev)) + return (dma_addr_t)sg_phys(sg); + + /* + * We can't use dma_map_sg, because we don't use scatterlists in + * the way it expects (we don't guarantee that the scatterlist + * will exist for the lifetime of the mapping). + */ + return dma_map_page(vring_dma_dev(vq), + sg_page(sg), sg->offset, sg->length, + direction); +} + +static dma_addr_t vring_map_single(const struct vring_virtqueue *vq, + void *cpu_addr, size_t size, + enum dma_data_direction direction) +{ + if (!vring_use_dma_api(vq->vq.vdev)) + return (dma_addr_t)virt_to_phys(cpu_addr); + + return dma_map_single(vring_dma_dev(vq), + cpu_addr, size, direction); +} + +static void vring_unmap_one(const struct vring_virtqueue *vq, + struct vring_desc *desc) +{ + u16 flags; + + if (!vring_use_dma_api(vq->vq.vdev)) + return; + + flags = virtio16_to_cpu(vq->vq.vdev, desc->flags); + + if (flags & VRING_DESC_F_INDIRECT) { + dma_unmap_single(vring_dma_dev(vq), + virtio64_to_cpu(vq->vq.vdev, desc->addr), + virtio32_to_cpu(vq->vq.vdev, desc->len), + (flags & VRING_DESC_F_WRITE) ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); + } else { + dma_unmap_page(vring_dma_dev(vq), + virtio64_to_cpu(vq->vq.vdev, desc->addr), + virtio32_to_cpu(vq->vq.vdev, desc->len), + (flags & VRING_DESC_F_WRITE) ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); + } +} + +static int vring_mapping_error(const struct vring_virtqueue *vq, + dma_addr_t addr) +{ + if (!vring_use_dma_api(vq->vq.vdev)) + return 0; + + return dma_mapping_error(vring_dma_dev(vq), addr); +} + static struct vring_desc *alloc_indirect(struct virtqueue *_vq, unsigned int total_sg, gfp_t gfp) { @@ -161,7 +240,7 @@ static inline int virtqueue_add(struct virtqueue *_vq, struct vring_virtqueue *vq = to_vvq(_vq); struct scatterlist *sg; struct vring_desc *desc; - unsigned int i, n, avail, descs_used, uninitialized_var(prev); + unsigned int i, n, avail, descs_used, uninitialized_var(prev), err_idx; int head; bool indirect; @@ -201,21 +280,15 @@ static inline int virtqueue_add(struct virtqueue *_vq, if (desc) { /* Use a single buffer which doesn't continue */ - vq->vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_INDIRECT); - vq->vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, virt_to_phys(desc)); - /* avoid kmemleak false positive (hidden by virt_to_phys) */ - kmemleak_ignore(desc); - vq->vring.desc[head].len = cpu_to_virtio32(_vq->vdev, total_sg * sizeof(struct vring_desc)); - + indirect = true; /* Set up rest to use this indirect table. */ i = 0; descs_used = 1; - indirect = true; } else { + indirect = false; desc = vq->vring.desc; i = head; descs_used = total_sg; - indirect = false; } if (vq->vq.num_free < descs_used) { @@ -230,13 +303,14 @@ static inline int virtqueue_add(struct virtqueue *_vq, return -ENOSPC; } - /* We're about to use some buffers from the free list. */ - vq->vq.num_free -= descs_used; - for (n = 0; n < out_sgs; n++) { for (sg = sgs[n]; sg; sg = sg_next(sg)) { + dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_TO_DEVICE); + if (vring_mapping_error(vq, addr)) + goto unmap_release; + desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT); - desc[i].addr = cpu_to_virtio64(_vq->vdev, sg_phys(sg)); + desc[i].addr = cpu_to_virtio64(_vq->vdev, addr); desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length); prev = i; i = virtio16_to_cpu(_vq->vdev, desc[i].next); @@ -244,8 +318,12 @@ static inline int virtqueue_add(struct virtqueue *_vq, } for (; n < (out_sgs + in_sgs); n++) { for (sg = sgs[n]; sg; sg = sg_next(sg)) { + dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_FROM_DEVICE); + if (vring_mapping_error(vq, addr)) + goto unmap_release; + desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT | VRING_DESC_F_WRITE); - desc[i].addr = cpu_to_virtio64(_vq->vdev, sg_phys(sg)); + desc[i].addr = cpu_to_virtio64(_vq->vdev, addr); desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length); prev = i; i = virtio16_to_cpu(_vq->vdev, desc[i].next); @@ -254,14 +332,33 @@ static inline int virtqueue_add(struct virtqueue *_vq, /* Last one doesn't continue. */ desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT); + if (indirect) { + /* Now that the indirect table is filled in, map it. */ + dma_addr_t addr = vring_map_single( + vq, desc, total_sg * sizeof(struct vring_desc), + DMA_TO_DEVICE); + if (vring_mapping_error(vq, addr)) + goto unmap_release; + + vq->vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_INDIRECT); + vq->vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, addr); + + vq->vring.desc[head].len = cpu_to_virtio32(_vq->vdev, total_sg * sizeof(struct vring_desc)); + } + + /* We're using some buffers from the free list. */ + vq->vq.num_free -= descs_used; + /* Update free pointer */ if (indirect) vq->free_head = virtio16_to_cpu(_vq->vdev, vq->vring.desc[head].next); else vq->free_head = i; - /* Set token. */ - vq->data[head] = data; + /* Store token and indirect buffer state. */ + vq->desc_state[head].data = data; + if (indirect) + vq->desc_state[head].indir_desc = desc; /* Put entry in available array (but don't update avail->idx until they * do sync). */ @@ -284,6 +381,24 @@ static inline int virtqueue_add(struct virtqueue *_vq, virtqueue_kick(_vq); return 0; + +unmap_release: + err_idx = i; + i = head; + + for (n = 0; n < total_sg; n++) { + if (i == err_idx) + break; + vring_unmap_one(vq, &desc[i]); + i = vq->vring.desc[i].next; + } + + vq->vq.num_free += total_sg; + + if (indirect) + kfree(desc); + + return -EIO; } /** @@ -454,27 +569,43 @@ EXPORT_SYMBOL_GPL(virtqueue_kick); static void detach_buf(struct vring_virtqueue *vq, unsigned int head) { - unsigned int i; + unsigned int i, j; + u16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT); /* Clear data ptr. */ - vq->data[head] = NULL; + vq->desc_state[head].data = NULL; - /* Put back on free list: find end */ + /* Put back on free list: unmap first-level descriptors and find end */ i = head; - /* Free the indirect table */ - if (vq->vring.desc[i].flags & cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT)) - kfree(phys_to_virt(virtio64_to_cpu(vq->vq.vdev, vq->vring.desc[i].addr))); - - while (vq->vring.desc[i].flags & cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT)) { + while (vq->vring.desc[i].flags & nextflag) { + vring_unmap_one(vq, &vq->vring.desc[i]); i = virtio16_to_cpu(vq->vq.vdev, vq->vring.desc[i].next); vq->vq.num_free++; } + vring_unmap_one(vq, &vq->vring.desc[i]); vq->vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev, vq->free_head); vq->free_head = head; + /* Plus final descriptor */ vq->vq.num_free++; + + /* Free the indirect table, if any, now that it's unmapped. */ + if (vq->desc_state[head].indir_desc) { + struct vring_desc *indir_desc = vq->desc_state[head].indir_desc; + u32 len = virtio32_to_cpu(vq->vq.vdev, vq->vring.desc[head].len); + + BUG_ON(!(vq->vring.desc[head].flags & + cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT))); + BUG_ON(len == 0 || len % sizeof(struct vring_desc)); + + for (j = 0; j < len / sizeof(struct vring_desc); j++) + vring_unmap_one(vq, &indir_desc[j]); + + kfree(vq->desc_state[head].indir_desc); + vq->desc_state[head].indir_desc = NULL; + } } static inline bool more_used(const struct vring_virtqueue *vq) @@ -529,13 +660,13 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len) BAD_RING(vq, "id %u out of range\n", i); return NULL; } - if (unlikely(!vq->data[i])) { + if (unlikely(!vq->desc_state[i].data)) { BAD_RING(vq, "id %u is not a head!\n", i); return NULL; } /* detach_buf clears data, so grab it now. */ - ret = vq->data[i]; + ret = vq->desc_state[i].data; detach_buf(vq, i); vq->last_used_idx++; /* If we expect an interrupt for the next entry, tell host @@ -709,10 +840,10 @@ void *virtqueue_detach_unused_buf(struct virtqueue *_vq) START_USE(vq); for (i = 0; i < vq->vring.num; i++) { - if (!vq->data[i]) + if (!vq->desc_state[i].data) continue; /* detach_buf clears data, so grab it now. */ - buf = vq->data[i]; + buf = vq->desc_state[i].data; detach_buf(vq, i); vq->avail_idx_shadow--; vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow); @@ -766,7 +897,8 @@ struct virtqueue *vring_new_virtqueue(unsigned int index, return NULL; } - vq = kmalloc(sizeof(*vq) + sizeof(void *)*num, GFP_KERNEL); + vq = kmalloc(sizeof(*vq) + num * sizeof(struct vring_desc_state), + GFP_KERNEL); if (!vq) return NULL; @@ -800,11 +932,9 @@ struct virtqueue *vring_new_virtqueue(unsigned int index, /* Put everything in free lists. */ vq->free_head = 0; - for (i = 0; i < num-1; i++) { + for (i = 0; i < num-1; i++) vq->vring.desc[i].next = cpu_to_virtio16(vdev, i + 1); - vq->data[i] = NULL; - } - vq->data[i] = NULL; + memset(vq->desc_state, 0, num * sizeof(struct vring_desc_state)); return &vq->vq; } diff --git a/tools/virtio/linux/dma-mapping.h b/tools/virtio/linux/dma-mapping.h new file mode 100644 index 000000000000..4f93af89ae16 --- /dev/null +++ b/tools/virtio/linux/dma-mapping.h @@ -0,0 +1,17 @@ +#ifndef _LINUX_DMA_MAPPING_H +#define _LINUX_DMA_MAPPING_H + +#ifdef CONFIG_HAS_DMA +# error Virtio userspace code does not support CONFIG_HAS_DMA +#endif + +#define PCI_DMA_BUS_IS_PHYS 1 + +enum dma_data_direction { + DMA_BIDIRECTIONAL = 0, + DMA_TO_DEVICE = 1, + DMA_FROM_DEVICE = 2, + DMA_NONE = 3, +}; + +#endif -- cgit v1.2.3 From 2a2d1382fe9dccfce6f9c60a9c9fd2f0fe5bcf2b Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Tue, 2 Feb 2016 21:46:37 -0800 Subject: virtio: Add improved queue allocation API This leaves vring_new_virtqueue alone for compatbility, but it adds two new improved APIs: vring_create_virtqueue: Creates a virtqueue backed by automatically allocated coherent memory. (Some day it this could be extended to support non-coherent memory, too, if there ends up being a platform on which it's worthwhile.) __vring_new_virtqueue: Creates a virtqueue with a manually-specified layout. This should allow mic_virtio to work much more cleanly. Signed-off-by: Andy Lutomirski Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 209 ++++++++++++++++++++++++++++++++++++------- include/linux/virtio.h | 23 ++++- include/linux/virtio_ring.h | 35 ++++++++ 3 files changed, 235 insertions(+), 32 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 9abc008ff7ea..e46d08107a50 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -95,6 +95,11 @@ struct vring_virtqueue { /* How to notify other side. FIXME: commonalize hcalls! */ bool (*notify)(struct virtqueue *vq); + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + #ifdef DEBUG /* They're supposed to lock for us. */ unsigned int in_use; @@ -878,36 +883,31 @@ irqreturn_t vring_interrupt(int irq, void *_vq) } EXPORT_SYMBOL_GPL(vring_interrupt); -struct virtqueue *vring_new_virtqueue(unsigned int index, - unsigned int num, - unsigned int vring_align, - struct virtio_device *vdev, - bool weak_barriers, - void *pages, - bool (*notify)(struct virtqueue *), - void (*callback)(struct virtqueue *), - const char *name) +struct virtqueue *__vring_new_virtqueue(unsigned int index, + struct vring vring, + struct virtio_device *vdev, + bool weak_barriers, + bool (*notify)(struct virtqueue *), + void (*callback)(struct virtqueue *), + const char *name) { - struct vring_virtqueue *vq; unsigned int i; + struct vring_virtqueue *vq; - /* We assume num is a power of 2. */ - if (num & (num - 1)) { - dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num); - return NULL; - } - - vq = kmalloc(sizeof(*vq) + num * sizeof(struct vring_desc_state), + vq = kmalloc(sizeof(*vq) + vring.num * sizeof(struct vring_desc_state), GFP_KERNEL); if (!vq) return NULL; - vring_init(&vq->vring, num, pages, vring_align); + vq->vring = vring; vq->vq.callback = callback; vq->vq.vdev = vdev; vq->vq.name = name; - vq->vq.num_free = num; + vq->vq.num_free = vring.num; vq->vq.index = index; + vq->we_own_ring = false; + vq->queue_dma_addr = 0; + vq->queue_size_in_bytes = 0; vq->notify = notify; vq->weak_barriers = weak_barriers; vq->broken = false; @@ -932,18 +932,145 @@ struct virtqueue *vring_new_virtqueue(unsigned int index, /* Put everything in free lists. */ vq->free_head = 0; - for (i = 0; i < num-1; i++) + for (i = 0; i < vring.num-1; i++) vq->vring.desc[i].next = cpu_to_virtio16(vdev, i + 1); - memset(vq->desc_state, 0, num * sizeof(struct vring_desc_state)); + memset(vq->desc_state, 0, vring.num * sizeof(struct vring_desc_state)); return &vq->vq; } +EXPORT_SYMBOL_GPL(__vring_new_virtqueue); + +static void *vring_alloc_queue(struct virtio_device *vdev, size_t size, + dma_addr_t *dma_handle, gfp_t flag) +{ + if (vring_use_dma_api(vdev)) { + return dma_alloc_coherent(vdev->dev.parent, size, + dma_handle, flag); + } else { + void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag); + if (queue) { + phys_addr_t phys_addr = virt_to_phys(queue); + *dma_handle = (dma_addr_t)phys_addr; + + /* + * Sanity check: make sure we dind't truncate + * the address. The only arches I can find that + * have 64-bit phys_addr_t but 32-bit dma_addr_t + * are certain non-highmem MIPS and x86 + * configurations, but these configurations + * should never allocate physical pages above 32 + * bits, so this is fine. Just in case, throw a + * warning and abort if we end up with an + * unrepresentable address. + */ + if (WARN_ON_ONCE(*dma_handle != phys_addr)) { + free_pages_exact(queue, PAGE_ALIGN(size)); + return NULL; + } + } + return queue; + } +} + +static void vring_free_queue(struct virtio_device *vdev, size_t size, + void *queue, dma_addr_t dma_handle) +{ + if (vring_use_dma_api(vdev)) { + dma_free_coherent(vdev->dev.parent, size, queue, dma_handle); + } else { + free_pages_exact(queue, PAGE_ALIGN(size)); + } +} + +struct virtqueue *vring_create_virtqueue( + unsigned int index, + unsigned int num, + unsigned int vring_align, + struct virtio_device *vdev, + bool weak_barriers, + bool may_reduce_num, + bool (*notify)(struct virtqueue *), + void (*callback)(struct virtqueue *), + const char *name) +{ + struct virtqueue *vq; + void *queue; + dma_addr_t dma_addr; + size_t queue_size_in_bytes; + struct vring vring; + + /* We assume num is a power of 2. */ + if (num & (num - 1)) { + dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num); + return NULL; + } + + /* TODO: allocate each queue chunk individually */ + for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) { + queue = vring_alloc_queue(vdev, vring_size(num, vring_align), + &dma_addr, + GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO); + if (queue) + break; + } + + if (!num) + return NULL; + + if (!queue) { + /* Try to get a single page. You are my only hope! */ + queue = vring_alloc_queue(vdev, vring_size(num, vring_align), + &dma_addr, GFP_KERNEL|__GFP_ZERO); + } + if (!queue) + return NULL; + + queue_size_in_bytes = vring_size(num, vring_align); + vring_init(&vring, num, queue, vring_align); + + vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, + notify, callback, name); + if (!vq) { + vring_free_queue(vdev, queue_size_in_bytes, queue, + dma_addr); + return NULL; + } + + to_vvq(vq)->queue_dma_addr = dma_addr; + to_vvq(vq)->queue_size_in_bytes = queue_size_in_bytes; + to_vvq(vq)->we_own_ring = true; + + return vq; +} +EXPORT_SYMBOL_GPL(vring_create_virtqueue); + +struct virtqueue *vring_new_virtqueue(unsigned int index, + unsigned int num, + unsigned int vring_align, + struct virtio_device *vdev, + bool weak_barriers, + void *pages, + bool (*notify)(struct virtqueue *vq), + void (*callback)(struct virtqueue *vq), + const char *name) +{ + struct vring vring; + vring_init(&vring, num, pages, vring_align); + return __vring_new_virtqueue(index, vring, vdev, weak_barriers, + notify, callback, name); +} EXPORT_SYMBOL_GPL(vring_new_virtqueue); -void vring_del_virtqueue(struct virtqueue *vq) +void vring_del_virtqueue(struct virtqueue *_vq) { - list_del(&vq->list); - kfree(to_vvq(vq)); + struct vring_virtqueue *vq = to_vvq(_vq); + + if (vq->we_own_ring) { + vring_free_queue(vq->vq.vdev, vq->queue_size_in_bytes, + vq->vring.desc, vq->queue_dma_addr); + } + list_del(&_vq->list); + kfree(vq); } EXPORT_SYMBOL_GPL(vring_del_virtqueue); @@ -1007,20 +1134,42 @@ void virtio_break_device(struct virtio_device *dev) } EXPORT_SYMBOL_GPL(virtio_break_device); -void *virtqueue_get_avail(struct virtqueue *_vq) +dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); - return vq->vring.avail; + BUG_ON(!vq->we_own_ring); + + return vq->queue_dma_addr; } -EXPORT_SYMBOL_GPL(virtqueue_get_avail); +EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr); -void *virtqueue_get_used(struct virtqueue *_vq) +dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); - return vq->vring.used; + BUG_ON(!vq->we_own_ring); + + return vq->queue_dma_addr + + ((char *)vq->vring.avail - (char *)vq->vring.desc); +} +EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr); + +dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + + BUG_ON(!vq->we_own_ring); + + return vq->queue_dma_addr + + ((char *)vq->vring.used - (char *)vq->vring.desc); +} +EXPORT_SYMBOL_GPL(virtqueue_get_used_addr); + +const struct vring *virtqueue_get_vring(struct virtqueue *vq) +{ + return &to_vvq(vq)->vring; } -EXPORT_SYMBOL_GPL(virtqueue_get_used); +EXPORT_SYMBOL_GPL(virtqueue_get_vring); MODULE_LICENSE("GPL"); diff --git a/include/linux/virtio.h b/include/linux/virtio.h index 8f4d4bfa6d46..d5eb5479a425 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -75,8 +75,27 @@ unsigned int virtqueue_get_vring_size(struct virtqueue *vq); bool virtqueue_is_broken(struct virtqueue *vq); -void *virtqueue_get_avail(struct virtqueue *vq); -void *virtqueue_get_used(struct virtqueue *vq); +const struct vring *virtqueue_get_vring(struct virtqueue *vq); +dma_addr_t virtqueue_get_desc_addr(struct virtqueue *vq); +dma_addr_t virtqueue_get_avail_addr(struct virtqueue *vq); +dma_addr_t virtqueue_get_used_addr(struct virtqueue *vq); + +/* + * Legacy accessors -- in almost all cases, these are the wrong functions + * to use. + */ +static inline void *virtqueue_get_desc(struct virtqueue *vq) +{ + return virtqueue_get_vring(vq)->desc; +} +static inline void *virtqueue_get_avail(struct virtqueue *vq) +{ + return virtqueue_get_vring(vq)->avail; +} +static inline void *virtqueue_get_used(struct virtqueue *vq) +{ + return virtqueue_get_vring(vq)->used; +} /** * virtio_device - representation of a device using virtio diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h index a156e2b6ccfe..e8d36938f09a 100644 --- a/include/linux/virtio_ring.h +++ b/include/linux/virtio_ring.h @@ -59,6 +59,35 @@ static inline void virtio_store_mb(bool weak_barriers, struct virtio_device; struct virtqueue; +/* + * Creates a virtqueue and allocates the descriptor ring. If + * may_reduce_num is set, then this may allocate a smaller ring than + * expected. The caller should query virtqueue_get_ring_size to learn + * the actual size of the ring. + */ +struct virtqueue *vring_create_virtqueue(unsigned int index, + unsigned int num, + unsigned int vring_align, + struct virtio_device *vdev, + bool weak_barriers, + bool may_reduce_num, + bool (*notify)(struct virtqueue *vq), + void (*callback)(struct virtqueue *vq), + const char *name); + +/* Creates a virtqueue with a custom layout. */ +struct virtqueue *__vring_new_virtqueue(unsigned int index, + struct vring vring, + struct virtio_device *vdev, + bool weak_barriers, + bool (*notify)(struct virtqueue *), + void (*callback)(struct virtqueue *), + const char *name); + +/* + * Creates a virtqueue with a standard layout but a caller-allocated + * ring. + */ struct virtqueue *vring_new_virtqueue(unsigned int index, unsigned int num, unsigned int vring_align, @@ -68,7 +97,13 @@ struct virtqueue *vring_new_virtqueue(unsigned int index, bool (*notify)(struct virtqueue *vq), void (*callback)(struct virtqueue *vq), const char *name); + +/* + * Destroys a virtqueue. If created with vring_create_virtqueue, this + * also frees the ring. + */ void vring_del_virtqueue(struct virtqueue *vq); + /* Filter out transport-specific feature bits. */ void vring_transport_features(struct virtio_device *vdev); -- cgit v1.2.3 From b42111382f0e677e2e227c5c4894423cbdaed1f1 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Tue, 2 Feb 2016 21:46:38 -0800 Subject: virtio_mmio: Use the DMA API if enabled This switches to vring_create_virtqueue, simplifying the driver and adding DMA API support. Signed-off-by: Andy Lutomirski Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_mmio.c | 67 ++++++++++---------------------------------- 1 file changed, 15 insertions(+), 52 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 745c6ee1bb3e..48bfea91dbca 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -99,12 +99,6 @@ struct virtio_mmio_vq_info { /* the actual virtqueue */ struct virtqueue *vq; - /* the number of entries in the queue */ - unsigned int num; - - /* the virtual address of the ring queue */ - void *queue; - /* the list node for the virtqueues list */ struct list_head node; }; @@ -322,15 +316,13 @@ static void vm_del_vq(struct virtqueue *vq) { struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vq->vdev); struct virtio_mmio_vq_info *info = vq->priv; - unsigned long flags, size; + unsigned long flags; unsigned int index = vq->index; spin_lock_irqsave(&vm_dev->lock, flags); list_del(&info->node); spin_unlock_irqrestore(&vm_dev->lock, flags); - vring_del_virtqueue(vq); - /* Select and deactivate the queue */ writel(index, vm_dev->base + VIRTIO_MMIO_QUEUE_SEL); if (vm_dev->version == 1) { @@ -340,8 +332,8 @@ static void vm_del_vq(struct virtqueue *vq) WARN_ON(readl(vm_dev->base + VIRTIO_MMIO_QUEUE_READY)); } - size = PAGE_ALIGN(vring_size(info->num, VIRTIO_MMIO_VRING_ALIGN)); - free_pages_exact(info->queue, size); + vring_del_virtqueue(vq); + kfree(info); } @@ -356,8 +348,6 @@ static void vm_del_vqs(struct virtio_device *vdev) free_irq(platform_get_irq(vm_dev->pdev, 0), vm_dev); } - - static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index, void (*callback)(struct virtqueue *vq), const char *name) @@ -365,7 +355,8 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index, struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); struct virtio_mmio_vq_info *info; struct virtqueue *vq; - unsigned long flags, size; + unsigned long flags; + unsigned int num; int err; if (!name) @@ -388,66 +379,40 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index, goto error_kmalloc; } - /* Allocate pages for the queue - start with a queue as big as - * possible (limited by maximum size allowed by device), drop down - * to a minimal size, just big enough to fit descriptor table - * and two rings (which makes it "alignment_size * 2") - */ - info->num = readl(vm_dev->base + VIRTIO_MMIO_QUEUE_NUM_MAX); - - /* If the device reports a 0 entry queue, we won't be able to - * use it to perform I/O, and vring_new_virtqueue() can't create - * empty queues anyway, so don't bother to set up the device. - */ - if (info->num == 0) { + num = readl(vm_dev->base + VIRTIO_MMIO_QUEUE_NUM_MAX); + if (num == 0) { err = -ENOENT; - goto error_alloc_pages; - } - - while (1) { - size = PAGE_ALIGN(vring_size(info->num, - VIRTIO_MMIO_VRING_ALIGN)); - /* Did the last iter shrink the queue below minimum size? */ - if (size < VIRTIO_MMIO_VRING_ALIGN * 2) { - err = -ENOMEM; - goto error_alloc_pages; - } - - info->queue = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO); - if (info->queue) - break; - - info->num /= 2; + goto error_new_virtqueue; } /* Create the vring */ - vq = vring_new_virtqueue(index, info->num, VIRTIO_MMIO_VRING_ALIGN, vdev, - true, info->queue, vm_notify, callback, name); + vq = vring_create_virtqueue(index, num, VIRTIO_MMIO_VRING_ALIGN, vdev, + true, true, vm_notify, callback, name); if (!vq) { err = -ENOMEM; goto error_new_virtqueue; } /* Activate the queue */ - writel(info->num, vm_dev->base + VIRTIO_MMIO_QUEUE_NUM); + writel(virtqueue_get_vring_size(vq), vm_dev->base + VIRTIO_MMIO_QUEUE_NUM); if (vm_dev->version == 1) { writel(PAGE_SIZE, vm_dev->base + VIRTIO_MMIO_QUEUE_ALIGN); - writel(virt_to_phys(info->queue) >> PAGE_SHIFT, + writel(virtqueue_get_desc_addr(vq) >> PAGE_SHIFT, vm_dev->base + VIRTIO_MMIO_QUEUE_PFN); } else { u64 addr; - addr = virt_to_phys(info->queue); + addr = virtqueue_get_desc_addr(vq); writel((u32)addr, vm_dev->base + VIRTIO_MMIO_QUEUE_DESC_LOW); writel((u32)(addr >> 32), vm_dev->base + VIRTIO_MMIO_QUEUE_DESC_HIGH); - addr = virt_to_phys(virtqueue_get_avail(vq)); + addr = virtqueue_get_avail_addr(vq); writel((u32)addr, vm_dev->base + VIRTIO_MMIO_QUEUE_AVAIL_LOW); writel((u32)(addr >> 32), vm_dev->base + VIRTIO_MMIO_QUEUE_AVAIL_HIGH); - addr = virt_to_phys(virtqueue_get_used(vq)); + addr = virtqueue_get_used_addr(vq); writel((u32)addr, vm_dev->base + VIRTIO_MMIO_QUEUE_USED_LOW); writel((u32)(addr >> 32), vm_dev->base + VIRTIO_MMIO_QUEUE_USED_HIGH); @@ -471,8 +436,6 @@ error_new_virtqueue: writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_READY); WARN_ON(readl(vm_dev->base + VIRTIO_MMIO_QUEUE_READY)); } - free_pages_exact(info->queue, size); -error_alloc_pages: kfree(info); error_kmalloc: error_available: -- cgit v1.2.3 From 7a5589b240b405d55b2b395554082ec284f414bb Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Tue, 2 Feb 2016 21:46:39 -0800 Subject: virtio_pci: Use the DMA API if enabled This switches to vring_create_virtqueue, simplifying the driver and adding DMA API support. This fixes virtio-pci on platforms and busses that have IOMMUs. This will break the experimental QEMU Q35 IOMMU support until QEMU is fixed. In exchange, it fixes physical virtio hardware as well as virtio-pci running under Xen. Signed-off-by: Andy Lutomirski Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_pci_common.h | 6 ---- drivers/virtio/virtio_pci_legacy.c | 42 +++++++++++--------------- drivers/virtio/virtio_pci_modern.c | 61 ++++++++++---------------------------- 3 files changed, 33 insertions(+), 76 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h index 2cc252270b2d..28263200ed42 100644 --- a/drivers/virtio/virtio_pci_common.h +++ b/drivers/virtio/virtio_pci_common.h @@ -35,12 +35,6 @@ struct virtio_pci_vq_info { /* the actual virtqueue */ struct virtqueue *vq; - /* the number of entries in the queue */ - int num; - - /* the virtual address of the ring queue */ - void *queue; - /* the list node for the virtqueues list */ struct list_head node; diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c index 48bc9797e530..8c4e61783441 100644 --- a/drivers/virtio/virtio_pci_legacy.c +++ b/drivers/virtio/virtio_pci_legacy.c @@ -119,7 +119,6 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, u16 msix_vec) { struct virtqueue *vq; - unsigned long size; u16 num; int err; @@ -131,27 +130,19 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, if (!num || ioread32(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN)) return ERR_PTR(-ENOENT); - info->num = num; info->msix_vector = msix_vec; - size = PAGE_ALIGN(vring_size(num, VIRTIO_PCI_VRING_ALIGN)); - info->queue = alloc_pages_exact(size, GFP_KERNEL|__GFP_ZERO); - if (info->queue == NULL) + /* create the vring */ + vq = vring_create_virtqueue(index, num, + VIRTIO_PCI_VRING_ALIGN, &vp_dev->vdev, + true, false, vp_notify, callback, name); + if (!vq) return ERR_PTR(-ENOMEM); /* activate the queue */ - iowrite32(virt_to_phys(info->queue) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT, + iowrite32(virtqueue_get_desc_addr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN); - /* create the vring */ - vq = vring_new_virtqueue(index, info->num, - VIRTIO_PCI_VRING_ALIGN, &vp_dev->vdev, - true, info->queue, vp_notify, callback, name); - if (!vq) { - err = -ENOMEM; - goto out_activate_queue; - } - vq->priv = (void __force *)vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NOTIFY; if (msix_vec != VIRTIO_MSI_NO_VECTOR) { @@ -159,17 +150,15 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, msix_vec = ioread16(vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR); if (msix_vec == VIRTIO_MSI_NO_VECTOR) { err = -EBUSY; - goto out_assign; + goto out_deactivate; } } return vq; -out_assign: - vring_del_virtqueue(vq); -out_activate_queue: +out_deactivate: iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN); - free_pages_exact(info->queue, size); + vring_del_virtqueue(vq); return ERR_PTR(err); } @@ -177,7 +166,6 @@ static void del_vq(struct virtio_pci_vq_info *info) { struct virtqueue *vq = info->vq; struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); - unsigned long size; iowrite16(vq->index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL); @@ -188,13 +176,10 @@ static void del_vq(struct virtio_pci_vq_info *info) ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR); } - vring_del_virtqueue(vq); - /* Select and deactivate the queue */ iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN); - size = PAGE_ALIGN(vring_size(info->num, VIRTIO_PCI_VRING_ALIGN)); - free_pages_exact(info->queue, size); + vring_del_virtqueue(vq); } static const struct virtio_config_ops virtio_pci_config_ops = { @@ -227,6 +212,13 @@ int virtio_pci_legacy_probe(struct virtio_pci_device *vp_dev) return -ENODEV; } + rc = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64)); + if (rc) + rc = dma_set_mask_and_coherent(&pci_dev->dev, + DMA_BIT_MASK(32)); + if (rc) + dev_warn(&pci_dev->dev, "Failed to enable 64-bit or 32-bit DMA. Trying to continue, but this might not work.\n"); + rc = pci_request_region(pci_dev, 0, "virtio-pci-legacy"); if (rc) return rc; diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index 7760fc1a2218..f6f28cc7eb45 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -287,31 +287,6 @@ static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) return vp_ioread16(&vp_dev->common->msix_config); } -static size_t vring_pci_size(u16 num) -{ - /* We only need a cacheline separation. */ - return PAGE_ALIGN(vring_size(num, SMP_CACHE_BYTES)); -} - -static void *alloc_virtqueue_pages(int *num) -{ - void *pages; - - /* TODO: allocate each queue chunk individually */ - for (; *num && vring_pci_size(*num) > PAGE_SIZE; *num /= 2) { - pages = alloc_pages_exact(vring_pci_size(*num), - GFP_KERNEL|__GFP_ZERO|__GFP_NOWARN); - if (pages) - return pages; - } - - if (!*num) - return NULL; - - /* Try to get a single page. You are my only hope! */ - return alloc_pages_exact(vring_pci_size(*num), GFP_KERNEL|__GFP_ZERO); -} - static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, struct virtio_pci_vq_info *info, unsigned index, @@ -343,29 +318,22 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, /* get offset of notification word for this vq */ off = vp_ioread16(&cfg->queue_notify_off); - info->num = num; info->msix_vector = msix_vec; - info->queue = alloc_virtqueue_pages(&info->num); - if (info->queue == NULL) - return ERR_PTR(-ENOMEM); - /* create the vring */ - vq = vring_new_virtqueue(index, info->num, - SMP_CACHE_BYTES, &vp_dev->vdev, - true, info->queue, vp_notify, callback, name); - if (!vq) { - err = -ENOMEM; - goto err_new_queue; - } + vq = vring_create_virtqueue(index, num, + SMP_CACHE_BYTES, &vp_dev->vdev, + true, true, vp_notify, callback, name); + if (!vq) + return ERR_PTR(-ENOMEM); /* activate the queue */ - vp_iowrite16(num, &cfg->queue_size); - vp_iowrite64_twopart(virt_to_phys(info->queue), + vp_iowrite16(virtqueue_get_vring_size(vq), &cfg->queue_size); + vp_iowrite64_twopart(virtqueue_get_desc_addr(vq), &cfg->queue_desc_lo, &cfg->queue_desc_hi); - vp_iowrite64_twopart(virt_to_phys(virtqueue_get_avail(vq)), + vp_iowrite64_twopart(virtqueue_get_avail_addr(vq), &cfg->queue_avail_lo, &cfg->queue_avail_hi); - vp_iowrite64_twopart(virt_to_phys(virtqueue_get_used(vq)), + vp_iowrite64_twopart(virtqueue_get_used_addr(vq), &cfg->queue_used_lo, &cfg->queue_used_hi); if (vp_dev->notify_base) { @@ -410,8 +378,6 @@ err_assign_vector: pci_iounmap(vp_dev->pci_dev, (void __iomem __force *)vq->priv); err_map_notify: vring_del_virtqueue(vq); -err_new_queue: - free_pages_exact(info->queue, vring_pci_size(info->num)); return ERR_PTR(err); } @@ -456,8 +422,6 @@ static void del_vq(struct virtio_pci_vq_info *info) pci_iounmap(vp_dev->pci_dev, (void __force __iomem *)vq->priv); vring_del_virtqueue(vq); - - free_pages_exact(info->queue, vring_pci_size(info->num)); } static const struct virtio_config_ops virtio_pci_config_nodev_ops = { @@ -641,6 +605,13 @@ int virtio_pci_modern_probe(struct virtio_pci_device *vp_dev) return -EINVAL; } + err = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64)); + if (err) + err = dma_set_mask_and_coherent(&pci_dev->dev, + DMA_BIT_MASK(32)); + if (err) + dev_warn(&pci_dev->dev, "Failed to enable 64-bit or 32-bit DMA. Trying to continue, but this might not work.\n"); + /* Device capability is only mandatory for devices that have * device-specific configuration. */ -- cgit v1.2.3 From 78fe39872378b0bef00a91181f1947acb8a08500 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Tue, 2 Feb 2016 21:46:40 -0800 Subject: vring: Use the DMA API on Xen Signed-off-by: Andy Lutomirski Signed-off-by: Michael S. Tsirkin Reviewed-by: David Vrabel Reviewed-by: Wei Liu --- drivers/virtio/virtio_ring.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index e46d08107a50..5c802d47892c 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -25,6 +25,7 @@ #include #include #include +#include #ifdef DEBUG /* For development, we want to crash whenever the ring is screwed. */ @@ -136,6 +137,17 @@ struct vring_virtqueue { static bool vring_use_dma_api(struct virtio_device *vdev) { + /* + * In theory, it's possible to have a buggy QEMU-supposed + * emulated Q35 IOMMU and Xen enabled at the same time. On + * such a configuration, virtio has never worked and will + * not work without an even larger kludge. Instead, enable + * the DMA API if we're a Xen guest, which at least allows + * all of the sensible Xen configurations to work correctly. + */ + if (xen_domain()) + return true; + return false; } -- cgit v1.2.3 From 592002f55e6e0b198a01d4a9315f74ddfea1c403 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Wed, 24 Feb 2016 17:07:27 +0200 Subject: virtio_blk: VIRTIO_BLK_F_WCE->VIRTIO_BLK_F_FLUSH Latest virtio spec says the feature bit name is VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_WCE is the legacy name. virtio blk header says exactly the reverse - fix that and update driver code to match. Cc: Paolo Bonzini Signed-off-by: Michael S. Tsirkin --- drivers/block/virtio_blk.c | 11 ++++++++--- include/uapi/linux/virtio_blk.h | 6 +++--- 2 files changed, 11 insertions(+), 6 deletions(-) (limited to 'drivers') diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 6ca35495a5be..28cff0d23d82 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -477,8 +477,13 @@ static int virtblk_get_cache_mode(struct virtio_device *vdev) err = virtio_cread_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE, struct virtio_blk_config, wce, &writeback); + + /* + * If WCE is not configurable and flush is not available, + * assume no writeback cache is in use. + */ if (err) - writeback = virtio_has_feature(vdev, VIRTIO_BLK_F_WCE); + writeback = virtio_has_feature(vdev, VIRTIO_BLK_F_FLUSH); return writeback; } @@ -833,14 +838,14 @@ static const struct virtio_device_id id_table[] = { static unsigned int features_legacy[] = { VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY, VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, VIRTIO_BLK_F_SCSI, - VIRTIO_BLK_F_WCE, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, + VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, VIRTIO_BLK_F_MQ, } ; static unsigned int features[] = { VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY, VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, - VIRTIO_BLK_F_WCE, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, + VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, VIRTIO_BLK_F_MQ, }; diff --git a/include/uapi/linux/virtio_blk.h b/include/uapi/linux/virtio_blk.h index 19c66fcbab8a..9ebe4d968dd5 100644 --- a/include/uapi/linux/virtio_blk.h +++ b/include/uapi/linux/virtio_blk.h @@ -43,11 +43,11 @@ #ifndef VIRTIO_BLK_NO_LEGACY #define VIRTIO_BLK_F_BARRIER 0 /* Does host support barriers? */ #define VIRTIO_BLK_F_SCSI 7 /* Supports scsi command passthru */ -#define VIRTIO_BLK_F_WCE 9 /* Writeback mode enabled after reset */ +#define VIRTIO_BLK_F_FLUSH 9 /* Flush command supported */ #define VIRTIO_BLK_F_CONFIG_WCE 11 /* Writeback mode available in config */ #ifndef __KERNEL__ -/* Old (deprecated) name for VIRTIO_BLK_F_WCE. */ -#define VIRTIO_BLK_F_FLUSH VIRTIO_BLK_F_WCE +/* Old (deprecated) name for VIRTIO_BLK_F_FLUSH. */ +#define VIRTIO_BLK_F_WCE VIRTIO_BLK_F_FLUSH #endif #endif /* !VIRTIO_BLK_NO_LEGACY */ -- cgit v1.2.3 From c507203756ca6303df2191c96c1385e965e2f0b7 Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Tue, 16 Feb 2016 15:59:34 +0100 Subject: vhost: rename cross-endian helpers The default use case for vhost is when the host and the vring have the same endianness (default native endianness). But there are cases where they differ and vhost should byteswap when accessing the vring. The first case is when the host is big endian and the vring belongs to a virtio 1.0 device, which is always little endian. This is covered by the vq->is_le field. This field is initialized when userspace calls the VHOST_SET_FEATURES ioctl. It is reset when the device stops. We already have a vhost_init_is_le() helper, but the reset operation is opencoded as follows: vq->is_le = virtio_legacy_is_little_endian(); It isn't clear that we are resetting vq->is_le here. This patch moves the code to a helper with a more explicit name. The other case where we may have to byteswap is when the architecture can switch endianness at runtime (bi-endian). If endianness differs in the host and in the guest, then legacy devices need to be used in cross-endian mode. This mode is available with CONFIG_VHOST_CROSS_ENDIAN_LEGACY=y, which introduces a vq->user_be field. Userspace may enable cross-endian mode by calling the SET_VRING_ENDIAN ioctl before the device is started. The cross-endian mode is disabled when the device is stopped. The current names of the helpers that manipulate vq->user_be are unclear. This patch renames those helpers to clearly show that this is cross-endian stuff and with explicit enable/disable semantics. No behaviour change. Signed-off-by: Greg Kurz Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vhost.c | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) (limited to 'drivers') diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 236553e81027..69f6463e11bd 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -43,11 +43,21 @@ enum { #define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num]) #ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY -static void vhost_vq_reset_user_be(struct vhost_virtqueue *vq) +static void vhost_disable_cross_endian(struct vhost_virtqueue *vq) { vq->user_be = !virtio_legacy_is_little_endian(); } +static void vhost_enable_cross_endian_big(struct vhost_virtqueue *vq) +{ + vq->user_be = true; +} + +static void vhost_enable_cross_endian_little(struct vhost_virtqueue *vq) +{ + vq->user_be = false; +} + static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp) { struct vhost_vring_state s; @@ -62,7 +72,10 @@ static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp) s.num != VHOST_VRING_BIG_ENDIAN) return -EINVAL; - vq->user_be = s.num; + if (s.num == VHOST_VRING_BIG_ENDIAN) + vhost_enable_cross_endian_big(vq); + else + vhost_enable_cross_endian_little(vq); return 0; } @@ -91,7 +104,7 @@ static void vhost_init_is_le(struct vhost_virtqueue *vq) vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) || !vq->user_be; } #else -static void vhost_vq_reset_user_be(struct vhost_virtqueue *vq) +static void vhost_disable_cross_endian(struct vhost_virtqueue *vq) { } @@ -113,6 +126,11 @@ static void vhost_init_is_le(struct vhost_virtqueue *vq) } #endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */ +static void vhost_reset_is_le(struct vhost_virtqueue *vq) +{ + vq->is_le = virtio_legacy_is_little_endian(); +} + static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh, poll_table *pt) { @@ -276,8 +294,8 @@ static void vhost_vq_reset(struct vhost_dev *dev, vq->call = NULL; vq->log_ctx = NULL; vq->memory = NULL; - vq->is_le = virtio_legacy_is_little_endian(); - vhost_vq_reset_user_be(vq); + vhost_reset_is_le(vq); + vhost_disable_cross_endian(vq); } static int vhost_worker(void *data) @@ -1159,7 +1177,7 @@ int vhost_init_used(struct vhost_virtqueue *vq) bool is_le = vq->is_le; if (!vq->private_data) { - vq->is_le = virtio_legacy_is_little_endian(); + vhost_reset_is_le(vq); return 0; } -- cgit v1.2.3 From 80f7d0301e7913f704d3505722f806717c61dff5 Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Tue, 16 Feb 2016 15:59:44 +0100 Subject: vhost: rename vhost_init_used() Looking at how callers use this, maybe we should just rename init_used to vhost_vq_init_access. The _used suffix was a hint that we access the vq used ring. But maybe what callers care about is that it must be called after access_ok. Also, this function manipulates the vq->is_le field which isn't related to the vq used ring. This patch simply renames vhost_init_used() to vhost_vq_init_access() as suggested by Michael. No behaviour change. Signed-off-by: Greg Kurz Signed-off-by: Michael S. Tsirkin --- drivers/vhost/net.c | 2 +- drivers/vhost/scsi.c | 2 +- drivers/vhost/test.c | 2 +- drivers/vhost/vhost.c | 4 ++-- drivers/vhost/vhost.h | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) (limited to 'drivers') diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 9eda69e40678..7bd75ff8be26 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -917,7 +917,7 @@ static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd) vhost_net_disable_vq(n, vq); vq->private_data = sock; - r = vhost_init_used(vq); + r = vhost_vq_init_access(vq); if (r) goto err_used; r = vhost_net_enable_vq(n, vq); diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c index 29cfc57d496e..f898686cdd93 100644 --- a/drivers/vhost/scsi.c +++ b/drivers/vhost/scsi.c @@ -1274,7 +1274,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs, vq = &vs->vqs[i].vq; mutex_lock(&vq->mutex); vq->private_data = vs_tpg; - vhost_init_used(vq); + vhost_vq_init_access(vq); mutex_unlock(&vq->mutex); } ret = 0; diff --git a/drivers/vhost/test.c b/drivers/vhost/test.c index f2882ac98726..388eec4e1a90 100644 --- a/drivers/vhost/test.c +++ b/drivers/vhost/test.c @@ -196,7 +196,7 @@ static long vhost_test_run(struct vhost_test *n, int test) oldpriv = vq->private_data; vq->private_data = priv; - r = vhost_init_used(&n->vqs[index]); + r = vhost_vq_init_access(&n->vqs[index]); mutex_unlock(&vq->mutex); diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 69f6463e11bd..328c54ab0154 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -1170,7 +1170,7 @@ static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event) return 0; } -int vhost_init_used(struct vhost_virtqueue *vq) +int vhost_vq_init_access(struct vhost_virtqueue *vq) { __virtio16 last_used_idx; int r; @@ -1200,7 +1200,7 @@ err: vq->is_le = is_le; return r; } -EXPORT_SYMBOL_GPL(vhost_init_used); +EXPORT_SYMBOL_GPL(vhost_vq_init_access); static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, struct iovec iov[], int iov_size) diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index d3f767448a72..8f0dd0d915d4 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -148,7 +148,7 @@ int vhost_get_vq_desc(struct vhost_virtqueue *, struct vhost_log *log, unsigned int *log_num); void vhost_discard_vq_desc(struct vhost_virtqueue *, int n); -int vhost_init_used(struct vhost_virtqueue *); +int vhost_vq_init_access(struct vhost_virtqueue *); int vhost_add_used(struct vhost_virtqueue *, unsigned int head, int len); int vhost_add_used_n(struct vhost_virtqueue *, struct vring_used_elem *heads, unsigned count); -- cgit v1.2.3 From 15e90f526bb4e1f9d9b3ecaa8e20bd608bd4dac7 Mon Sep 17 00:00:00 2001 From: Geliang Tang Date: Tue, 1 Mar 2016 13:44:52 +0100 Subject: virtio/s390: use dev_to_virtio Use dev_to_virtio() instead of open-coding it. Signed-off-by: Geliang Tang Message-Id: <912bf59bd3a48f2d4d4994681e898dc084fe29d3.1451484163.git.geliangtang@163.com> Signed-off-by: Cornelia Huck Signed-off-by: Michael S. Tsirkin --- drivers/s390/virtio/virtio_ccw.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index bf2d1300a957..46b110a124b0 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c @@ -945,8 +945,7 @@ static struct virtio_config_ops virtio_ccw_config_ops = { static void virtio_ccw_release_dev(struct device *_d) { - struct virtio_device *dev = container_of(_d, struct virtio_device, - dev); + struct virtio_device *dev = dev_to_virtio(_d); struct virtio_ccw_device *vcdev = to_vc_device(dev); kfree(vcdev->status); -- cgit v1.2.3 From c235082b50f6dd81436b82fd73ffbdeede4ec2e4 Mon Sep 17 00:00:00 2001 From: Cornelia Huck Date: Tue, 1 Mar 2016 13:44:53 +0100 Subject: virtio/s390: size of SET_IND payload SET_IND takes as a payload the _address_ of the indicators, meaning that we have one of the rare cases where kmalloc(sizeof(&...)) is actually correct. Let's clarify that with a comment. The count for the ccw, however, was only correct because the indicators are 64 bit. Let's use the correct value. Reported-by: Martin Schwidefsky Reviewed-by: Halil Pasic Reviewed-by: David Hildenbrand Signed-off-by: Cornelia Huck Signed-off-by: Michael S. Tsirkin --- drivers/s390/virtio/virtio_ccw.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index 46b110a124b0..8688ad4c825f 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c @@ -342,13 +342,14 @@ static void virtio_ccw_drop_indicator(struct virtio_ccw_device *vcdev, ccw->count = sizeof(*thinint_area); ccw->cda = (__u32)(unsigned long) thinint_area; } else { + /* payload is the address of the indicators */ indicatorp = kmalloc(sizeof(&vcdev->indicators), GFP_DMA | GFP_KERNEL); if (!indicatorp) return; *indicatorp = 0; ccw->cmd_code = CCW_CMD_SET_IND; - ccw->count = sizeof(vcdev->indicators); + ccw->count = sizeof(&vcdev->indicators); ccw->cda = (__u32)(unsigned long) indicatorp; } /* Deregister indicators from host. */ @@ -656,7 +657,10 @@ static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs, } } ret = -ENOMEM; - /* We need a data area under 2G to communicate. */ + /* + * We need a data area under 2G to communicate. Our payload is + * the address of the indicators. + */ indicatorp = kmalloc(sizeof(&vcdev->indicators), GFP_DMA | GFP_KERNEL); if (!indicatorp) goto out; @@ -672,7 +676,7 @@ static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs, vcdev->indicators = 0; ccw->cmd_code = CCW_CMD_SET_IND; ccw->flags = 0; - ccw->count = sizeof(vcdev->indicators); + ccw->count = sizeof(&vcdev->indicators); ccw->cda = (__u32)(unsigned long) indicatorp; ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_SET_IND); if (ret) @@ -683,7 +687,7 @@ static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs, vcdev->indicators2 = 0; ccw->cmd_code = CCW_CMD_SET_CONF_IND; ccw->flags = 0; - ccw->count = sizeof(vcdev->indicators2); + ccw->count = sizeof(&vcdev->indicators2); ccw->cda = (__u32)(unsigned long) indicatorp; ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_SET_CONF_IND); if (ret) -- cgit v1.2.3 From fad7b7b27b6a168ca8ebc84482043886f837b89d Mon Sep 17 00:00:00 2001 From: Petr Mladek Date: Mon, 25 Jan 2016 17:38:05 +0100 Subject: virtio_balloon: Use a workqueue instead of "vballoon" kthread This patch moves the deferred work from the "vballoon" kthread into a system freezable workqueue. We do not need to maintain and run a dedicated kthread. Also the event driven workqueues API makes the logic much easier. Especially, we do not longer need an own wait queue, wait function, and freeze point. The conversion is pretty straightforward. One cycle of the main loop is put into a work. The work is queued instead of waking the kthread. fill_balloon() and leak_balloon() have a limit for the amount of modified pages. The work re-queues itself when necessary. For this, we make fill_balloon() to return the number of really modified pages. Note that leak_balloon() already did this. virtballoon_restore() queues the work only when really needed. The only complication is that we need to prevent queuing the work when the balloon is being removed. It was easier before because the kthread simply removed itself from the wait queue. We need an extra boolean and spin lock now. My initial idea was to use a dedicated workqueue. Michael S. Tsirkin suggested using a system one. Tejun Heo confirmed that the system workqueue has a pretty high concurrency level (256) by default. Therefore we need not be afraid of too long blocking. Signed-off-by: Petr Mladek Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_balloon.c | 108 +++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 57 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 0c3691f46575..2c9a92f1e525 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -22,8 +22,7 @@ #include #include #include -#include -#include +#include #include #include #include @@ -49,11 +48,12 @@ struct virtio_balloon { struct virtio_device *vdev; struct virtqueue *inflate_vq, *deflate_vq, *stats_vq; - /* Where the ballooning thread waits for config to change. */ - wait_queue_head_t config_change; + /* The balloon servicing is delegated to a freezable workqueue. */ + struct work_struct work; - /* The thread servicing the balloon. */ - struct task_struct *thread; + /* Prevent updating balloon when it is being canceled. */ + spinlock_t stop_update_lock; + bool stop_update; /* Waiting for host to ack the pages we released. */ wait_queue_head_t acked; @@ -135,9 +135,10 @@ static void set_page_pfns(u32 pfns[], struct page *page) pfns[i] = page_to_balloon_pfn(page) + i; } -static void fill_balloon(struct virtio_balloon *vb, size_t num) +static unsigned fill_balloon(struct virtio_balloon *vb, size_t num) { struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info; + unsigned num_allocated_pages; /* We can only do one array worth at a time. */ num = min(num, ARRAY_SIZE(vb->pfns)); @@ -162,10 +163,13 @@ static void fill_balloon(struct virtio_balloon *vb, size_t num) adjust_managed_page_count(page, -1); } + num_allocated_pages = vb->num_pfns; /* Did we get any? */ if (vb->num_pfns != 0) tell_host(vb, vb->inflate_vq); mutex_unlock(&vb->balloon_lock); + + return num_allocated_pages; } static void release_pages_balloon(struct virtio_balloon *vb) @@ -251,14 +255,19 @@ static void update_balloon_stats(struct virtio_balloon *vb) * with a single buffer. From that point forward, all conversations consist of * a hypervisor request (a call to this function) which directs us to refill * the virtqueue with a fresh stats buffer. Since stats collection can sleep, - * we notify our kthread which does the actual work via stats_handle_request(). + * we delegate the job to a freezable workqueue that will do the actual work via + * stats_handle_request(). */ static void stats_request(struct virtqueue *vq) { struct virtio_balloon *vb = vq->vdev->priv; vb->need_stats_update = 1; - wake_up(&vb->config_change); + + spin_lock(&vb->stop_update_lock); + if (!vb->stop_update) + queue_work(system_freezable_wq, &vb->work); + spin_unlock(&vb->stop_update_lock); } static void stats_handle_request(struct virtio_balloon *vb) @@ -281,8 +290,12 @@ static void stats_handle_request(struct virtio_balloon *vb) static void virtballoon_changed(struct virtio_device *vdev) { struct virtio_balloon *vb = vdev->priv; + unsigned long flags; - wake_up(&vb->config_change); + spin_lock_irqsave(&vb->stop_update_lock, flags); + if (!vb->stop_update) + queue_work(system_freezable_wq, &vb->work); + spin_unlock_irqrestore(&vb->stop_update_lock, flags); } static inline s64 towards_target(struct virtio_balloon *vb) @@ -345,43 +358,25 @@ static int virtballoon_oom_notify(struct notifier_block *self, return NOTIFY_OK; } -static int balloon(void *_vballoon) +static void balloon(struct work_struct *work) { - struct virtio_balloon *vb = _vballoon; - DEFINE_WAIT_FUNC(wait, woken_wake_function); - - set_freezable(); - while (!kthread_should_stop()) { - s64 diff; - - try_to_freeze(); - - add_wait_queue(&vb->config_change, &wait); - for (;;) { - if ((diff = towards_target(vb)) != 0 || - vb->need_stats_update || - kthread_should_stop() || - freezing(current)) - break; - wait_woken(&wait, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); - } - remove_wait_queue(&vb->config_change, &wait); + struct virtio_balloon *vb; + s64 diff; - if (vb->need_stats_update) - stats_handle_request(vb); - if (diff > 0) - fill_balloon(vb, diff); - else if (diff < 0) - leak_balloon(vb, -diff); - update_balloon_size(vb); + vb = container_of(work, struct virtio_balloon, work); + diff = towards_target(vb); - /* - * For large balloon changes, we could spend a lot of time - * and always have work to do. Be nice if preempt disabled. - */ - cond_resched(); - } - return 0; + if (vb->need_stats_update) + stats_handle_request(vb); + + if (diff > 0) + diff -= fill_balloon(vb, diff); + else if (diff < 0) + diff += leak_balloon(vb, -diff); + update_balloon_size(vb); + + if (diff) + queue_work(system_freezable_wq, work); } static int init_vqs(struct virtio_balloon *vb) @@ -499,9 +494,11 @@ static int virtballoon_probe(struct virtio_device *vdev) goto out; } + INIT_WORK(&vb->work, balloon); + spin_lock_init(&vb->stop_update_lock); + vb->stop_update = false; vb->num_pages = 0; mutex_init(&vb->balloon_lock); - init_waitqueue_head(&vb->config_change); init_waitqueue_head(&vb->acked); vb->vdev = vdev; vb->need_stats_update = 0; @@ -523,16 +520,8 @@ static int virtballoon_probe(struct virtio_device *vdev) virtio_device_ready(vdev); - vb->thread = kthread_run(balloon, vb, "vballoon"); - if (IS_ERR(vb->thread)) { - err = PTR_ERR(vb->thread); - goto out_del_vqs; - } - return 0; -out_del_vqs: - unregister_oom_notifier(&vb->nb); out_oom_notify: vdev->config->del_vqs(vdev); out_free_vb: @@ -559,7 +548,12 @@ static void virtballoon_remove(struct virtio_device *vdev) struct virtio_balloon *vb = vdev->priv; unregister_oom_notifier(&vb->nb); - kthread_stop(vb->thread); + + spin_lock_irq(&vb->stop_update_lock); + vb->stop_update = true; + spin_unlock_irq(&vb->stop_update_lock); + cancel_work_sync(&vb->work); + remove_common(vb); kfree(vb); } @@ -570,10 +564,9 @@ static int virtballoon_freeze(struct virtio_device *vdev) struct virtio_balloon *vb = vdev->priv; /* - * The kthread is already frozen by the PM core before this + * The workqueue is already frozen by the PM core before this * function is called. */ - remove_common(vb); return 0; } @@ -589,7 +582,8 @@ static int virtballoon_restore(struct virtio_device *vdev) virtio_device_ready(vdev); - fill_balloon(vb, towards_target(vb)); + if (towards_target(vb)) + virtballoon_changed(vdev); update_balloon_size(vb); return 0; } -- cgit v1.2.3 From fd0e21c31e1e6383af978239d07ced6e4e5d82c3 Mon Sep 17 00:00:00 2001 From: Petr Mladek Date: Mon, 25 Jan 2016 17:38:06 +0100 Subject: virtio_balloon: Allow to resize and update the balloon stats in parallel The virtio balloon statistics are not updated when the balloon is being resized. But it seems that both tasks could be done in parallel. stats_handle_request() updates the statistics in the balloon structure and then communicates with the host. update_balloon_stats() calls all_vm_events() that just reads some per-CPU variables. The values might change during and after the call but it is expected and happens even without this patch. update_balloon_stats() also calls si_meminfo(). It is a bit more complex function. It too just reads some variables and looks lock-less safe. In each case, it seems to be called lock-less on several similar locations, e.g. from post_status() in dm_thread_func(), or from vmballoon_send_get_target(). The communication with the host is done via a separate virtqueue, see vb->stats_vq vs. vb->inflate_vq and vb->deflate_vq. Therefore it could be used in parallel with fill_balloon() and leak_balloon(). This patch splits the existing work into two pieces. One is for updating the balloon stats. The other is for resizing of the balloon. It seems that they can be proceed in parallel without any extra locking. Signed-off-by: Petr Mladek Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_balloon.c | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 2c9a92f1e525..9057cc768ca5 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -49,7 +49,8 @@ struct virtio_balloon { struct virtqueue *inflate_vq, *deflate_vq, *stats_vq; /* The balloon servicing is delegated to a freezable workqueue. */ - struct work_struct work; + struct work_struct update_balloon_stats_work; + struct work_struct update_balloon_size_work; /* Prevent updating balloon when it is being canceled. */ spinlock_t stop_update_lock; @@ -76,7 +77,6 @@ struct virtio_balloon { u32 pfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX]; /* Memory statistics */ - int need_stats_update; struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR]; /* To register callback in oom notifier call chain */ @@ -123,6 +123,7 @@ static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq) /* When host has read buffer, this completes via balloon_ack */ wait_event(vb->acked, virtqueue_get_buf(vq, &len)); + } static void set_page_pfns(u32 pfns[], struct page *page) @@ -262,11 +263,9 @@ static void stats_request(struct virtqueue *vq) { struct virtio_balloon *vb = vq->vdev->priv; - vb->need_stats_update = 1; - spin_lock(&vb->stop_update_lock); if (!vb->stop_update) - queue_work(system_freezable_wq, &vb->work); + queue_work(system_freezable_wq, &vb->update_balloon_stats_work); spin_unlock(&vb->stop_update_lock); } @@ -276,7 +275,6 @@ static void stats_handle_request(struct virtio_balloon *vb) struct scatterlist sg; unsigned int len; - vb->need_stats_update = 0; update_balloon_stats(vb); vq = vb->stats_vq; @@ -294,7 +292,7 @@ static void virtballoon_changed(struct virtio_device *vdev) spin_lock_irqsave(&vb->stop_update_lock, flags); if (!vb->stop_update) - queue_work(system_freezable_wq, &vb->work); + queue_work(system_freezable_wq, &vb->update_balloon_size_work); spin_unlock_irqrestore(&vb->stop_update_lock, flags); } @@ -358,17 +356,24 @@ static int virtballoon_oom_notify(struct notifier_block *self, return NOTIFY_OK; } -static void balloon(struct work_struct *work) +static void update_balloon_stats_func(struct work_struct *work) +{ + struct virtio_balloon *vb; + + vb = container_of(work, struct virtio_balloon, + update_balloon_stats_work); + stats_handle_request(vb); +} + +static void update_balloon_size_func(struct work_struct *work) { struct virtio_balloon *vb; s64 diff; - vb = container_of(work, struct virtio_balloon, work); + vb = container_of(work, struct virtio_balloon, + update_balloon_size_work); diff = towards_target(vb); - if (vb->need_stats_update) - stats_handle_request(vb); - if (diff > 0) diff -= fill_balloon(vb, diff); else if (diff < 0) @@ -494,14 +499,14 @@ static int virtballoon_probe(struct virtio_device *vdev) goto out; } - INIT_WORK(&vb->work, balloon); + INIT_WORK(&vb->update_balloon_stats_work, update_balloon_stats_func); + INIT_WORK(&vb->update_balloon_size_work, update_balloon_size_func); spin_lock_init(&vb->stop_update_lock); vb->stop_update = false; vb->num_pages = 0; mutex_init(&vb->balloon_lock); init_waitqueue_head(&vb->acked); vb->vdev = vdev; - vb->need_stats_update = 0; balloon_devinfo_init(&vb->vb_dev_info); #ifdef CONFIG_BALLOON_COMPACTION @@ -552,7 +557,8 @@ static void virtballoon_remove(struct virtio_device *vdev) spin_lock_irq(&vb->stop_update_lock); vb->stop_update = true; spin_unlock_irq(&vb->stop_update_lock); - cancel_work_sync(&vb->work); + cancel_work_sync(&vb->update_balloon_size_work); + cancel_work_sync(&vb->update_balloon_stats_work); remove_common(vb); kfree(vb); -- cgit v1.2.3 From 526d3e7ff515582eaf7cf5d9da2a11bf681204e6 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 4 Mar 2016 06:24:51 -0500 Subject: vhost: introduce vhost_has_work() This path introduces a helper which can give a hint for whether or not there's a work queued in the work list. This could be used for busy polling code to exit the busy loop. Signed-off-by: Jason Wang Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vhost.c | 7 +++++++ drivers/vhost/vhost.h | 1 + 2 files changed, 8 insertions(+) (limited to 'drivers') diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 328c54ab0154..596de212fe7e 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -263,6 +263,13 @@ void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work) } EXPORT_SYMBOL_GPL(vhost_work_queue); +/* A lockless hint for busy polling code to exit the loop */ +bool vhost_has_work(struct vhost_dev *dev) +{ + return !list_empty(&dev->work_list); +} +EXPORT_SYMBOL_GPL(vhost_has_work); + void vhost_poll_queue(struct vhost_poll *poll) { vhost_work_queue(poll->dev, &poll->work); diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index 8f0dd0d915d4..eef33b11a7ed 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -37,6 +37,7 @@ struct vhost_poll { void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn); void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work); +bool vhost_has_work(struct vhost_dev *dev); void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, unsigned long mask, struct vhost_dev *dev); -- cgit v1.2.3 From d4a60603fa0b42012decfa058dfa44cffde7a10c Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 4 Mar 2016 06:24:52 -0500 Subject: vhost: introduce vhost_vq_avail_empty() This patch introduces a helper which will return true if we're sure that the available ring is empty for a specific vq. When we're not sure, e.g vq access failure, return false instead. This could be used for busy polling code to exit the busy loop. Signed-off-by: Jason Wang Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vhost.c | 14 ++++++++++++++ drivers/vhost/vhost.h | 1 + 2 files changed, 15 insertions(+) (limited to 'drivers') diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 596de212fe7e..97f26f0aab40 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -1658,6 +1658,20 @@ void vhost_add_used_and_signal_n(struct vhost_dev *dev, } EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n); +/* return true if we're sure that avaiable ring is empty */ +bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq) +{ + __virtio16 avail_idx; + int r; + + r = __get_user(avail_idx, &vq->avail->idx); + if (r) + return false; + + return vhost16_to_cpu(vq, avail_idx) == vq->avail_idx; +} +EXPORT_SYMBOL_GPL(vhost_vq_avail_empty); + /* OK, now we need to know about added descriptors. */ bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) { diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index eef33b11a7ed..af5af773bf7a 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -159,6 +159,7 @@ void vhost_add_used_and_signal_n(struct vhost_dev *, struct vhost_virtqueue *, struct vring_used_elem *heads, unsigned count); void vhost_signal(struct vhost_dev *, struct vhost_virtqueue *); void vhost_disable_notify(struct vhost_dev *, struct vhost_virtqueue *); +bool vhost_vq_avail_empty(struct vhost_dev *, struct vhost_virtqueue *); bool vhost_enable_notify(struct vhost_dev *, struct vhost_virtqueue *); int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, -- cgit v1.2.3 From 0308813724606549436d30efd877a80c8e00790e Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 4 Mar 2016 06:24:53 -0500 Subject: vhost_net: basic polling support This patch tries to poll for new added tx buffer or socket receive queue for a while at the end of tx/rx processing. The maximum time spent on polling were specified through a new kind of vring ioctl. Signed-off-by: Jason Wang Signed-off-by: Michael S. Tsirkin --- drivers/vhost/net.c | 78 +++++++++++++++++++++++++++++++++++++++++++--- drivers/vhost/vhost.c | 14 +++++++++ drivers/vhost/vhost.h | 1 + include/uapi/linux/vhost.h | 6 ++++ 4 files changed, 94 insertions(+), 5 deletions(-) (limited to 'drivers') diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 7bd75ff8be26..f744eeb3e2b4 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -287,6 +287,43 @@ static void vhost_zerocopy_callback(struct ubuf_info *ubuf, bool success) rcu_read_unlock_bh(); } +static inline unsigned long busy_clock(void) +{ + return local_clock() >> 10; +} + +static bool vhost_can_busy_poll(struct vhost_dev *dev, + unsigned long endtime) +{ + return likely(!need_resched()) && + likely(!time_after(busy_clock(), endtime)) && + likely(!signal_pending(current)) && + !vhost_has_work(dev); +} + +static int vhost_net_tx_get_vq_desc(struct vhost_net *net, + struct vhost_virtqueue *vq, + struct iovec iov[], unsigned int iov_size, + unsigned int *out_num, unsigned int *in_num) +{ + unsigned long uninitialized_var(endtime); + int r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov), + out_num, in_num, NULL, NULL); + + if (r == vq->num && vq->busyloop_timeout) { + preempt_disable(); + endtime = busy_clock() + vq->busyloop_timeout; + while (vhost_can_busy_poll(vq->dev, endtime) && + vhost_vq_avail_empty(vq->dev, vq)) + cpu_relax_lowlatency(); + preempt_enable(); + r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov), + out_num, in_num, NULL, NULL); + } + + return r; +} + /* Expects to be always run from workqueue - which acts as * read-size critical section for our kind of RCU. */ static void handle_tx(struct vhost_net *net) @@ -331,10 +368,9 @@ static void handle_tx(struct vhost_net *net) % UIO_MAXIOV == nvq->done_idx)) break; - head = vhost_get_vq_desc(vq, vq->iov, - ARRAY_SIZE(vq->iov), - &out, &in, - NULL, NULL); + head = vhost_net_tx_get_vq_desc(net, vq, vq->iov, + ARRAY_SIZE(vq->iov), + &out, &in); /* On error, stop handling until the next kick. */ if (unlikely(head < 0)) break; @@ -435,6 +471,38 @@ static int peek_head_len(struct sock *sk) return len; } +static int vhost_net_rx_peek_head_len(struct vhost_net *net, struct sock *sk) +{ + struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX]; + struct vhost_virtqueue *vq = &nvq->vq; + unsigned long uninitialized_var(endtime); + int len = peek_head_len(sk); + + if (!len && vq->busyloop_timeout) { + /* Both tx vq and rx socket were polled here */ + mutex_lock(&vq->mutex); + vhost_disable_notify(&net->dev, vq); + + preempt_disable(); + endtime = busy_clock() + vq->busyloop_timeout; + + while (vhost_can_busy_poll(&net->dev, endtime) && + skb_queue_empty(&sk->sk_receive_queue) && + vhost_vq_avail_empty(&net->dev, vq)) + cpu_relax_lowlatency(); + + preempt_enable(); + + if (vhost_enable_notify(&net->dev, vq)) + vhost_poll_queue(&vq->poll); + mutex_unlock(&vq->mutex); + + len = peek_head_len(sk); + } + + return len; +} + /* This is a multi-buffer version of vhost_get_desc, that works if * vq has read descriptors only. * @vq - the relevant virtqueue @@ -553,7 +621,7 @@ static void handle_rx(struct vhost_net *net) vq->log : NULL; mergeable = vhost_has_feature(vq, VIRTIO_NET_F_MRG_RXBUF); - while ((sock_len = peek_head_len(sock->sk))) { + while ((sock_len = vhost_net_rx_peek_head_len(net, sock->sk))) { sock_len += sock_hlen; vhost_len = sock_len + vhost_hlen; headcount = get_rx_bufs(vq, vq->heads, vhost_len, diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 97f26f0aab40..669fef1e2bb6 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -303,6 +303,7 @@ static void vhost_vq_reset(struct vhost_dev *dev, vq->memory = NULL; vhost_reset_is_le(vq); vhost_disable_cross_endian(vq); + vq->busyloop_timeout = 0; } static int vhost_worker(void *data) @@ -937,6 +938,19 @@ long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp) case VHOST_GET_VRING_ENDIAN: r = vhost_get_vring_endian(vq, idx, argp); break; + case VHOST_SET_VRING_BUSYLOOP_TIMEOUT: + if (copy_from_user(&s, argp, sizeof(s))) { + r = -EFAULT; + break; + } + vq->busyloop_timeout = s.num; + break; + case VHOST_GET_VRING_BUSYLOOP_TIMEOUT: + s.index = idx; + s.num = vq->busyloop_timeout; + if (copy_to_user(argp, &s, sizeof(s))) + r = -EFAULT; + break; default: r = -ENOIOCTLCMD; } diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index af5af773bf7a..d36d8beb3351 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -115,6 +115,7 @@ struct vhost_virtqueue { /* Ring endianness requested by userspace for cross-endian support. */ bool user_be; #endif + u32 busyloop_timeout; }; struct vhost_dev { diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h index ab3731917bac..61a8777178c6 100644 --- a/include/uapi/linux/vhost.h +++ b/include/uapi/linux/vhost.h @@ -126,6 +126,12 @@ struct vhost_memory { #define VHOST_SET_VRING_CALL _IOW(VHOST_VIRTIO, 0x21, struct vhost_vring_file) /* Set eventfd to signal an error */ #define VHOST_SET_VRING_ERR _IOW(VHOST_VIRTIO, 0x22, struct vhost_vring_file) +/* Set busy loop timeout (in us) */ +#define VHOST_SET_VRING_BUSYLOOP_TIMEOUT _IOW(VHOST_VIRTIO, 0x23, \ + struct vhost_vring_state) +/* Get busy loop timeout (in us) */ +#define VHOST_GET_VRING_BUSYLOOP_TIMEOUT _IOW(VHOST_VIRTIO, 0x24, \ + struct vhost_vring_state) /* VHOST_NET specific defines */ -- cgit v1.2.3 From c67f5db82027ba6d2ea4ac9176bc45996a03ae6a Mon Sep 17 00:00:00 2001 From: Paolo Abeni Date: Thu, 17 Mar 2016 15:44:00 +0100 Subject: virtio_net: replace netdev_alloc_skb_ip_align() with napi_alloc_skb() This gives small but noticeable rx performance improvement (2-3%) and will allow exploiting future napi improvement. Signed-off-by: Paolo Abeni Signed-off-by: Michael S. Tsirkin --- drivers/net/virtio_net.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 767ab11a6e9f..a43631696bcb 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -256,7 +256,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, p = page_address(page) + offset; /* copy small packet so we can reuse these pages for small data */ - skb = netdev_alloc_skb_ip_align(vi->dev, GOOD_COPY_LEN); + skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN); if (unlikely(!skb)) return NULL; -- cgit v1.2.3