diff options
| -rw-r--r-- | drivers/virtio/virtio_ring.c | 525 |
1 files changed, 271 insertions, 254 deletions
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 29fab2fb39cb..7cd40a2a0d21 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -113,6 +113,11 @@ struct vring_virtqueue { struct vring_desc_state desc_state[]; }; + +/* + * Helpers. + */ + #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) /* @@ -200,6 +205,20 @@ static dma_addr_t vring_map_single(const struct vring_virtqueue *vq, cpu_addr, size, direction); } +static int vring_mapping_error(const struct vring_virtqueue *vq, + dma_addr_t addr) +{ + if (!vring_use_dma_api(vq->vq.vdev)) + return 0; + + return dma_mapping_error(vring_dma_dev(vq), addr); +} + + +/* + * Split ring specific functions - *_split(). + */ + static void vring_unmap_one_split(const struct vring_virtqueue *vq, struct vring_desc *desc) { @@ -225,15 +244,6 @@ static void vring_unmap_one_split(const struct vring_virtqueue *vq, } } -static int vring_mapping_error(const struct vring_virtqueue *vq, - dma_addr_t addr) -{ - if (!vring_use_dma_api(vq->vq.vdev)) - return 0; - - return dma_mapping_error(vring_dma_dev(vq), addr); -} - static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq, unsigned int total_sg, gfp_t gfp) @@ -435,6 +445,256 @@ unmap_release: return -EIO; } +static bool virtqueue_kick_prepare_split(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + u16 new, old; + bool needs_kick; + + START_USE(vq); + /* We need to expose available array entries before checking avail + * event. */ + virtio_mb(vq->weak_barriers); + + old = vq->avail_idx_shadow - vq->num_added; + new = vq->avail_idx_shadow; + vq->num_added = 0; + +#ifdef DEBUG + if (vq->last_add_time_valid) { + WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), + vq->last_add_time)) > 100); + } + vq->last_add_time_valid = false; +#endif + + if (vq->event) { + needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev, vring_avail_event(&vq->vring)), + new, old); + } else { + needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(_vq->vdev, VRING_USED_F_NO_NOTIFY)); + } + END_USE(vq); + return needs_kick; +} + +static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head, + void **ctx) +{ + unsigned int i, j; + __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT); + + /* Clear data ptr. */ + vq->desc_state[head].data = NULL; + + /* Put back on free list: unmap first-level descriptors and find end */ + i = head; + + while (vq->vring.desc[i].flags & nextflag) { + vring_unmap_one_split(vq, &vq->vring.desc[i]); + i = virtio16_to_cpu(vq->vq.vdev, vq->vring.desc[i].next); + vq->vq.num_free++; + } + + vring_unmap_one_split(vq, &vq->vring.desc[i]); + vq->vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev, vq->free_head); + vq->free_head = head; + + /* Plus final descriptor */ + vq->vq.num_free++; + + if (vq->indirect) { + struct vring_desc *indir_desc = vq->desc_state[head].indir_desc; + u32 len; + + /* Free the indirect table, if any, now that it's unmapped. */ + if (!indir_desc) + return; + + len = virtio32_to_cpu(vq->vq.vdev, vq->vring.desc[head].len); + + BUG_ON(!(vq->vring.desc[head].flags & + cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT))); + BUG_ON(len == 0 || len % sizeof(struct vring_desc)); + + for (j = 0; j < len / sizeof(struct vring_desc); j++) + vring_unmap_one_split(vq, &indir_desc[j]); + + kfree(indir_desc); + vq->desc_state[head].indir_desc = NULL; + } else if (ctx) { + *ctx = vq->desc_state[head].indir_desc; + } +} + +static inline bool more_used_split(const struct vring_virtqueue *vq) +{ + return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, vq->vring.used->idx); +} + +static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq, + unsigned int *len, + void **ctx) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + void *ret; + unsigned int i; + u16 last_used; + + START_USE(vq); + + if (unlikely(vq->broken)) { + END_USE(vq); + return NULL; + } + + if (!more_used_split(vq)) { + pr_debug("No more buffers in queue\n"); + END_USE(vq); + return NULL; + } + + /* Only get used array entries after they have been exposed by host. */ + virtio_rmb(vq->weak_barriers); + + last_used = (vq->last_used_idx & (vq->vring.num - 1)); + i = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].id); + *len = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].len); + + if (unlikely(i >= vq->vring.num)) { + BAD_RING(vq, "id %u out of range\n", i); + return NULL; + } + if (unlikely(!vq->desc_state[i].data)) { + BAD_RING(vq, "id %u is not a head!\n", i); + return NULL; + } + + /* detach_buf_split clears data, so grab it now. */ + ret = vq->desc_state[i].data; + detach_buf_split(vq, i, ctx); + vq->last_used_idx++; + /* If we expect an interrupt for the next entry, tell host + * by writing event index and flush out the write before + * the read in the next get_buf call. */ + if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) + virtio_store_mb(vq->weak_barriers, + &vring_used_event(&vq->vring), + cpu_to_virtio16(_vq->vdev, vq->last_used_idx)); + +#ifdef DEBUG + vq->last_add_time_valid = false; +#endif + + END_USE(vq); + return ret; +} + +static void virtqueue_disable_cb_split(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + + if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) { + vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; + if (!vq->event) + vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); + } +} + +static unsigned virtqueue_enable_cb_prepare_split(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + u16 last_used_idx; + + START_USE(vq); + + /* We optimistically turn back on interrupts, then check if there was + * more to do. */ + /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to + * either clear the flags bit or point the event index at the next + * entry. Always do both to keep code simple. */ + if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { + vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; + if (!vq->event) + vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); + } + vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx); + END_USE(vq); + return last_used_idx; +} + +static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned last_used_idx) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + + return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev, + vq->vring.used->idx); +} + +static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + u16 bufs; + + START_USE(vq); + + /* We optimistically turn back on interrupts, then check if there was + * more to do. */ + /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to + * either clear the flags bit or point the event index at the next + * entry. Always update the event index to keep code simple. */ + if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { + vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; + if (!vq->event) + vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); + } + /* TODO: tune this threshold */ + bufs = (u16)(vq->avail_idx_shadow - vq->last_used_idx) * 3 / 4; + + virtio_store_mb(vq->weak_barriers, + &vring_used_event(&vq->vring), + cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs)); + + if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) { + END_USE(vq); + return false; + } + + END_USE(vq); + return true; +} + +static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + unsigned int i; + void *buf; + + START_USE(vq); + + for (i = 0; i < vq->vring.num; i++) { + if (!vq->desc_state[i].data) + continue; + /* detach_buf_split clears data, so grab it now. */ + buf = vq->desc_state[i].data; + detach_buf_split(vq, i, NULL); + vq->avail_idx_shadow--; + vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow); + END_USE(vq); + return buf; + } + /* That should have freed everything. */ + BUG_ON(vq->vq.num_free != vq->vring.num); + + END_USE(vq); + return NULL; +} + + +/* + * Generic functions and exported symbols. + */ + static inline int virtqueue_add(struct virtqueue *_vq, struct scatterlist *sgs[], unsigned int total_sg, @@ -474,6 +734,7 @@ int virtqueue_add_sgs(struct virtqueue *_vq, /* Count them first. */ for (i = 0; i < out_sgs + in_sgs; i++) { struct scatterlist *sg; + for (sg = sgs[i]; sg; sg = sg_next(sg)) total_sg++; } @@ -550,39 +811,6 @@ int virtqueue_add_inbuf_ctx(struct virtqueue *vq, } EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_ctx); -static bool virtqueue_kick_prepare_split(struct virtqueue *_vq) -{ - struct vring_virtqueue *vq = to_vvq(_vq); - u16 new, old; - bool needs_kick; - - START_USE(vq); - /* We need to expose available array entries before checking avail - * event. */ - virtio_mb(vq->weak_barriers); - - old = vq->avail_idx_shadow - vq->num_added; - new = vq->avail_idx_shadow; - vq->num_added = 0; - -#ifdef DEBUG - if (vq->last_add_time_valid) { - WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), - vq->last_add_time)) > 100); - } - vq->last_add_time_valid = false; -#endif - - if (vq->event) { - needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev, vring_avail_event(&vq->vring)), - new, old); - } else { - needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(_vq->vdev, VRING_USED_F_NO_NOTIFY)); - } - END_USE(vq); - return needs_kick; -} - /** * virtqueue_kick_prepare - first half of split virtqueue_kick call. * @vq: the struct virtqueue @@ -644,118 +872,6 @@ bool virtqueue_kick(struct virtqueue *vq) } EXPORT_SYMBOL_GPL(virtqueue_kick); -static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head, - void **ctx) -{ - unsigned int i, j; - __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT); - - /* Clear data ptr. */ - vq->desc_state[head].data = NULL; - - /* Put back on free list: unmap first-level descriptors and find end */ - i = head; - - while (vq->vring.desc[i].flags & nextflag) { - vring_unmap_one_split(vq, &vq->vring.desc[i]); - i = virtio16_to_cpu(vq->vq.vdev, vq->vring.desc[i].next); - vq->vq.num_free++; - } - - vring_unmap_one_split(vq, &vq->vring.desc[i]); - vq->vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev, vq->free_head); - vq->free_head = head; - - /* Plus final descriptor */ - vq->vq.num_free++; - - if (vq->indirect) { - struct vring_desc *indir_desc = vq->desc_state[head].indir_desc; - u32 len; - - /* Free the indirect table, if any, now that it's unmapped. */ - if (!indir_desc) - return; - - len = virtio32_to_cpu(vq->vq.vdev, vq->vring.desc[head].len); - - BUG_ON(!(vq->vring.desc[head].flags & - cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT))); - BUG_ON(len == 0 || len % sizeof(struct vring_desc)); - - for (j = 0; j < len / sizeof(struct vring_desc); j++) - vring_unmap_one_split(vq, &indir_desc[j]); - - kfree(indir_desc); - vq->desc_state[head].indir_desc = NULL; - } else if (ctx) { - *ctx = vq->desc_state[head].indir_desc; - } -} - -static inline bool more_used_split(const struct vring_virtqueue *vq) -{ - return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, vq->vring.used->idx); -} - -static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq, - unsigned int *len, - void **ctx) -{ - struct vring_virtqueue *vq = to_vvq(_vq); - void *ret; - unsigned int i; - u16 last_used; - - START_USE(vq); - - if (unlikely(vq->broken)) { - END_USE(vq); - return NULL; - } - - if (!more_used_split(vq)) { - pr_debug("No more buffers in queue\n"); - END_USE(vq); - return NULL; - } - - /* Only get used array entries after they have been exposed by host. */ - virtio_rmb(vq->weak_barriers); - - last_used = (vq->last_used_idx & (vq->vring.num - 1)); - i = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].id); - *len = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].len); - - if (unlikely(i >= vq->vring.num)) { - BAD_RING(vq, "id %u out of range\n", i); - return NULL; - } - if (unlikely(!vq->desc_state[i].data)) { - BAD_RING(vq, "id %u is not a head!\n", i); - return NULL; - } - - /* detach_buf_split clears data, so grab it now. */ - ret = vq->desc_state[i].data; - detach_buf_split(vq, i, ctx); - vq->last_used_idx++; - /* If we expect an interrupt for the next entry, tell host - * by writing event index and flush out the write before - * the read in the next get_buf call. */ - if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) - virtio_store_mb(vq->weak_barriers, - &vring_used_event(&vq->vring), - cpu_to_virtio16(_vq->vdev, vq->last_used_idx)); - -#ifdef DEBUG - vq->last_add_time_valid = false; -#endif - - END_USE(vq); - return ret; -} - /** * virtqueue_get_buf - get the next used buffer * @vq: the struct virtqueue we're talking about. @@ -785,17 +901,6 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len) } EXPORT_SYMBOL_GPL(virtqueue_get_buf); -static void virtqueue_disable_cb_split(struct virtqueue *_vq) -{ - struct vring_virtqueue *vq = to_vvq(_vq); - - if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) { - vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; - if (!vq->event) - vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); - } -} - /** * virtqueue_disable_cb - disable callbacks * @vq: the struct virtqueue we're talking about. @@ -811,28 +916,6 @@ void virtqueue_disable_cb(struct virtqueue *_vq) } EXPORT_SYMBOL_GPL(virtqueue_disable_cb); -static unsigned virtqueue_enable_cb_prepare_split(struct virtqueue *_vq) -{ - struct vring_virtqueue *vq = to_vvq(_vq); - u16 last_used_idx; - - START_USE(vq); - - /* We optimistically turn back on interrupts, then check if there was - * more to do. */ - /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to - * either clear the flags bit or point the event index at the next - * entry. Always do both to keep code simple. */ - if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { - vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; - if (!vq->event) - vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); - } - vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx); - END_USE(vq); - return last_used_idx; -} - /** * virtqueue_enable_cb_prepare - restart callbacks after disable_cb * @vq: the struct virtqueue we're talking about. @@ -851,14 +934,6 @@ unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq) } EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare); -static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned last_used_idx) -{ - struct vring_virtqueue *vq = to_vvq(_vq); - - return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev, - vq->vring.used->idx); -} - /** * virtqueue_poll - query pending used buffers * @vq: the struct virtqueue we're talking about. @@ -891,43 +966,11 @@ EXPORT_SYMBOL_GPL(virtqueue_poll); bool virtqueue_enable_cb(struct virtqueue *_vq) { unsigned last_used_idx = virtqueue_enable_cb_prepare(_vq); + return !virtqueue_poll(_vq, last_used_idx); } EXPORT_SYMBOL_GPL(virtqueue_enable_cb); -static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq) -{ - struct vring_virtqueue *vq = to_vvq(_vq); - u16 bufs; - - START_USE(vq); - - /* We optimistically turn back on interrupts, then check if there was - * more to do. */ - /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to - * either clear the flags bit or point the event index at the next - * entry. Always update the event index to keep code simple. */ - if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { - vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; - if (!vq->event) - vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); - } - /* TODO: tune this threshold */ - bufs = (u16)(vq->avail_idx_shadow - vq->last_used_idx) * 3 / 4; - - virtio_store_mb(vq->weak_barriers, - &vring_used_event(&vq->vring), - cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs)); - - if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) { - END_USE(vq); - return false; - } - - END_USE(vq); - return true; -} - /** * virtqueue_enable_cb_delayed - restart callbacks after disable_cb. * @vq: the struct virtqueue we're talking about. @@ -947,32 +990,6 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq) } EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed); -static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq) -{ - struct vring_virtqueue *vq = to_vvq(_vq); - unsigned int i; - void *buf; - - START_USE(vq); - - for (i = 0; i < vq->vring.num; i++) { - if (!vq->desc_state[i].data) - continue; - /* detach_buf_split clears data, so grab it now. */ - buf = vq->desc_state[i].data; - detach_buf_split(vq, i, NULL); - vq->avail_idx_shadow--; - vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow); - END_USE(vq); - return buf; - } - /* That should have freed everything. */ - BUG_ON(vq->vq.num_free != vq->vring.num); - - END_USE(vq); - return NULL; -} - /** * virtqueue_detach_unused_buf - detach first unused buffer * @vq: the struct virtqueue we're talking about. |