diff options
Diffstat (limited to 'drivers/gpu/drm/virtio/virtgpu_vq.c')
-rw-r--r-- | drivers/gpu/drm/virtio/virtgpu_vq.c | 369 |
1 files changed, 154 insertions, 215 deletions
diff --git a/drivers/gpu/drm/virtio/virtgpu_vq.c b/drivers/gpu/drm/virtio/virtgpu_vq.c index 5914e79d3429..73854915ec34 100644 --- a/drivers/gpu/drm/virtio/virtgpu_vq.c +++ b/drivers/gpu/drm/virtio/virtgpu_vq.c @@ -95,7 +95,8 @@ virtio_gpu_get_vbuf(struct virtio_gpu_device *vgdev, if (!vbuf) return ERR_PTR(-ENOMEM); - BUG_ON(size > MAX_INLINE_CMD_SIZE); + BUG_ON(size > MAX_INLINE_CMD_SIZE || + size < sizeof(struct virtio_gpu_ctrl_hdr)); vbuf->buf = (void *)vbuf + sizeof(*vbuf); vbuf->size = size; @@ -109,21 +110,14 @@ virtio_gpu_get_vbuf(struct virtio_gpu_device *vgdev, return vbuf; } -static void *virtio_gpu_alloc_cmd(struct virtio_gpu_device *vgdev, - struct virtio_gpu_vbuffer **vbuffer_p, - int size) +static struct virtio_gpu_ctrl_hdr * +virtio_gpu_vbuf_ctrl_hdr(struct virtio_gpu_vbuffer *vbuf) { - struct virtio_gpu_vbuffer *vbuf; - - vbuf = virtio_gpu_get_vbuf(vgdev, size, - sizeof(struct virtio_gpu_ctrl_hdr), - NULL, NULL); - if (IS_ERR(vbuf)) { - *vbuffer_p = NULL; - return ERR_CAST(vbuf); - } - *vbuffer_p = vbuf; - return vbuf->buf; + /* this assumes a vbuf contains a command that starts with a + * virtio_gpu_ctrl_hdr, which is true for both ctrl and cursor + * virtqueues. + */ + return (struct virtio_gpu_ctrl_hdr *)vbuf->buf; } static struct virtio_gpu_update_cursor* @@ -161,6 +155,25 @@ static void *virtio_gpu_alloc_cmd_resp(struct virtio_gpu_device *vgdev, return (struct virtio_gpu_command *)vbuf->buf; } +static void *virtio_gpu_alloc_cmd(struct virtio_gpu_device *vgdev, + struct virtio_gpu_vbuffer **vbuffer_p, + int size) +{ + return virtio_gpu_alloc_cmd_resp(vgdev, NULL, vbuffer_p, size, + sizeof(struct virtio_gpu_ctrl_hdr), + NULL); +} + +static void *virtio_gpu_alloc_cmd_cb(struct virtio_gpu_device *vgdev, + struct virtio_gpu_vbuffer **vbuffer_p, + int size, + virtio_gpu_resp_cb cb) +{ + return virtio_gpu_alloc_cmd_resp(vgdev, cb, vbuffer_p, size, + sizeof(struct virtio_gpu_ctrl_hdr), + NULL); +} + static void free_vbuf(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer *vbuf) { @@ -209,12 +222,12 @@ void virtio_gpu_dequeue_ctrl_func(struct work_struct *work) trace_virtio_gpu_cmd_response(vgdev->ctrlq.vq, resp); if (resp->type != cpu_to_le32(VIRTIO_GPU_RESP_OK_NODATA)) { - if (resp->type >= cpu_to_le32(VIRTIO_GPU_RESP_ERR_UNSPEC)) { + if (le32_to_cpu(resp->type) >= VIRTIO_GPU_RESP_ERR_UNSPEC) { struct virtio_gpu_ctrl_hdr *cmd; - cmd = (struct virtio_gpu_ctrl_hdr *)entry->buf; - DRM_ERROR("response 0x%x (command 0x%x)\n", - le32_to_cpu(resp->type), - le32_to_cpu(cmd->type)); + cmd = virtio_gpu_vbuf_ctrl_hdr(entry); + DRM_ERROR_RATELIMITED("response 0x%x (command 0x%x)\n", + le32_to_cpu(resp->type), + le32_to_cpu(cmd->type)); } else DRM_DEBUG("response 0x%x\n", le32_to_cpu(resp->type)); } @@ -307,109 +320,107 @@ static struct sg_table *vmalloc_to_sgt(char *data, uint32_t size, int *sg_ents) return sgt; } -static bool virtio_gpu_queue_ctrl_buffer_locked(struct virtio_gpu_device *vgdev, - struct virtio_gpu_vbuffer *vbuf, - struct scatterlist *vout) - __releases(&vgdev->ctrlq.qlock) - __acquires(&vgdev->ctrlq.qlock) +static void virtio_gpu_queue_ctrl_sgs(struct virtio_gpu_device *vgdev, + struct virtio_gpu_vbuffer *vbuf, + struct virtio_gpu_fence *fence, + int elemcnt, + struct scatterlist **sgs, + int outcnt, + int incnt) { struct virtqueue *vq = vgdev->ctrlq.vq; - struct scatterlist *sgs[3], vcmd, vresp; - int outcnt = 0, incnt = 0; - bool notify = false; - int ret; + int ret, idx; - if (!vgdev->vqs_ready) - return notify; + if (!drm_dev_enter(vgdev->ddev, &idx)) { + if (fence && vbuf->objs) + virtio_gpu_array_unlock_resv(vbuf->objs); + free_vbuf(vgdev, vbuf); + return; + } - sg_init_one(&vcmd, vbuf->buf, vbuf->size); - sgs[outcnt + incnt] = &vcmd; - outcnt++; + if (vgdev->has_indirect) + elemcnt = 1; - if (vout) { - sgs[outcnt + incnt] = vout; - outcnt++; +again: + spin_lock(&vgdev->ctrlq.qlock); + + if (vq->num_free < elemcnt) { + spin_unlock(&vgdev->ctrlq.qlock); + virtio_gpu_notify(vgdev); + wait_event(vgdev->ctrlq.ack_queue, vq->num_free >= elemcnt); + goto again; } - if (vbuf->resp_size) { - sg_init_one(&vresp, vbuf->resp_buf, vbuf->resp_size); - sgs[outcnt + incnt] = &vresp; - incnt++; + /* now that the position of the vbuf in the virtqueue is known, we can + * finally set the fence id + */ + if (fence) { + virtio_gpu_fence_emit(vgdev, virtio_gpu_vbuf_ctrl_hdr(vbuf), + fence); + if (vbuf->objs) { + virtio_gpu_array_add_fence(vbuf->objs, &fence->f); + virtio_gpu_array_unlock_resv(vbuf->objs); + } } -retry: ret = virtqueue_add_sgs(vq, sgs, outcnt, incnt, vbuf, GFP_ATOMIC); - if (ret == -ENOSPC) { - spin_unlock(&vgdev->ctrlq.qlock); - wait_event(vgdev->ctrlq.ack_queue, vq->num_free >= outcnt + incnt); - spin_lock(&vgdev->ctrlq.qlock); - goto retry; - } else { - trace_virtio_gpu_cmd_queue(vq, - (struct virtio_gpu_ctrl_hdr *)vbuf->buf); + WARN_ON(ret); - notify = virtqueue_kick_prepare(vq); - } - return notify; + trace_virtio_gpu_cmd_queue(vq, virtio_gpu_vbuf_ctrl_hdr(vbuf)); + + atomic_inc(&vgdev->pending_commands); + + spin_unlock(&vgdev->ctrlq.qlock); + + drm_dev_exit(idx); } static void virtio_gpu_queue_fenced_ctrl_buffer(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer *vbuf, - struct virtio_gpu_ctrl_hdr *hdr, struct virtio_gpu_fence *fence) { - struct virtqueue *vq = vgdev->ctrlq.vq; - struct scatterlist *vout = NULL, sg; + struct scatterlist *sgs[3], vcmd, vout, vresp; struct sg_table *sgt = NULL; - bool notify; - int outcnt = 0; + int elemcnt = 0, outcnt = 0, incnt = 0; + /* set up vcmd */ + sg_init_one(&vcmd, vbuf->buf, vbuf->size); + elemcnt++; + sgs[outcnt] = &vcmd; + outcnt++; + + /* set up vout */ if (vbuf->data_size) { if (is_vmalloc_addr(vbuf->data_buf)) { + int sg_ents; sgt = vmalloc_to_sgt(vbuf->data_buf, vbuf->data_size, - &outcnt); - if (!sgt) + &sg_ents); + if (!sgt) { + if (fence && vbuf->objs) + virtio_gpu_array_unlock_resv(vbuf->objs); return; - vout = sgt->sgl; + } + + elemcnt += sg_ents; + sgs[outcnt] = sgt->sgl; } else { - sg_init_one(&sg, vbuf->data_buf, vbuf->data_size); - vout = &sg; - outcnt = 1; + sg_init_one(&vout, vbuf->data_buf, vbuf->data_size); + elemcnt++; + sgs[outcnt] = &vout; } + outcnt++; } -again: - spin_lock(&vgdev->ctrlq.qlock); - - /* - * Make sure we have enouth space in the virtqueue. If not - * wait here until we have. - * - * Without that virtio_gpu_queue_ctrl_buffer_nolock might have - * to wait for free space, which can result in fence ids being - * submitted out-of-order. - */ - if (vq->num_free < 2 + outcnt) { - spin_unlock(&vgdev->ctrlq.qlock); - wait_event(vgdev->ctrlq.ack_queue, vq->num_free >= 3); - goto again; + /* set up vresp */ + if (vbuf->resp_size) { + sg_init_one(&vresp, vbuf->resp_buf, vbuf->resp_size); + elemcnt++; + sgs[outcnt + incnt] = &vresp; + incnt++; } - if (hdr && fence) { - virtio_gpu_fence_emit(vgdev, hdr, fence); - if (vbuf->objs) { - virtio_gpu_array_add_fence(vbuf->objs, &fence->f); - virtio_gpu_array_unlock_resv(vbuf->objs); - } - } - notify = virtio_gpu_queue_ctrl_buffer_locked(vgdev, vbuf, vout); - spin_unlock(&vgdev->ctrlq.qlock); - if (notify) { - if (vgdev->disable_notify) - vgdev->pending_notify = true; - else - virtqueue_notify(vgdev->ctrlq.vq); - } + virtio_gpu_queue_ctrl_sgs(vgdev, vbuf, fence, elemcnt, sgs, outcnt, + incnt); if (sgt) { sg_free_table(sgt); @@ -417,25 +428,26 @@ again: } } -void virtio_gpu_disable_notify(struct virtio_gpu_device *vgdev) +void virtio_gpu_notify(struct virtio_gpu_device *vgdev) { - vgdev->disable_notify = true; -} - -void virtio_gpu_enable_notify(struct virtio_gpu_device *vgdev) -{ - vgdev->disable_notify = false; + bool notify; - if (!vgdev->pending_notify) + if (!atomic_read(&vgdev->pending_commands)) return; - vgdev->pending_notify = false; - virtqueue_notify(vgdev->ctrlq.vq); + + spin_lock(&vgdev->ctrlq.qlock); + atomic_set(&vgdev->pending_commands, 0); + notify = virtqueue_kick_prepare(vgdev->ctrlq.vq); + spin_unlock(&vgdev->ctrlq.qlock); + + if (notify) + virtqueue_notify(vgdev->ctrlq.vq); } static void virtio_gpu_queue_ctrl_buffer(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer *vbuf) { - virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, NULL, NULL); + virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, NULL); } static void virtio_gpu_queue_cursor(struct virtio_gpu_device *vgdev, @@ -443,12 +455,13 @@ static void virtio_gpu_queue_cursor(struct virtio_gpu_device *vgdev, { struct virtqueue *vq = vgdev->cursorq.vq; struct scatterlist *sgs[1], ccmd; + int idx, ret, outcnt; bool notify; - int ret; - int outcnt; - if (!vgdev->vqs_ready) + if (!drm_dev_enter(vgdev->ddev, &idx)) { + free_vbuf(vgdev, vbuf); return; + } sg_init_one(&ccmd, vbuf->buf, vbuf->size); sgs[0] = &ccmd; @@ -464,7 +477,7 @@ retry: goto retry; } else { trace_virtio_gpu_cmd_queue(vq, - (struct virtio_gpu_ctrl_hdr *)vbuf->buf); + virtio_gpu_vbuf_ctrl_hdr(vbuf)); notify = virtqueue_kick_prepare(vq); } @@ -473,6 +486,8 @@ retry: if (notify) virtqueue_notify(vq); + + drm_dev_exit(idx); } /* just create gem objects for userspace and long lived objects, @@ -499,39 +514,36 @@ void virtio_gpu_cmd_create_resource(struct virtio_gpu_device *vgdev, cmd_p->width = cpu_to_le32(params->width); cmd_p->height = cpu_to_le32(params->height); - virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, &cmd_p->hdr, fence); + virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence); bo->created = true; } -void virtio_gpu_cmd_unref_resource(struct virtio_gpu_device *vgdev, - uint32_t resource_id) +static void virtio_gpu_cmd_unref_cb(struct virtio_gpu_device *vgdev, + struct virtio_gpu_vbuffer *vbuf) { - struct virtio_gpu_resource_unref *cmd_p; - struct virtio_gpu_vbuffer *vbuf; + struct virtio_gpu_object *bo; - cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p)); - memset(cmd_p, 0, sizeof(*cmd_p)); + bo = vbuf->resp_cb_data; + vbuf->resp_cb_data = NULL; - cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_RESOURCE_UNREF); - cmd_p->resource_id = cpu_to_le32(resource_id); - - virtio_gpu_queue_ctrl_buffer(vgdev, vbuf); + virtio_gpu_cleanup_object(bo); } -static void virtio_gpu_cmd_resource_inval_backing(struct virtio_gpu_device *vgdev, - uint32_t resource_id, - struct virtio_gpu_fence *fence) +void virtio_gpu_cmd_unref_resource(struct virtio_gpu_device *vgdev, + struct virtio_gpu_object *bo) { - struct virtio_gpu_resource_detach_backing *cmd_p; + struct virtio_gpu_resource_unref *cmd_p; struct virtio_gpu_vbuffer *vbuf; - cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p)); + cmd_p = virtio_gpu_alloc_cmd_cb(vgdev, &vbuf, sizeof(*cmd_p), + virtio_gpu_cmd_unref_cb); memset(cmd_p, 0, sizeof(*cmd_p)); - cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_RESOURCE_DETACH_BACKING); - cmd_p->resource_id = cpu_to_le32(resource_id); + cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_RESOURCE_UNREF); + cmd_p->resource_id = cpu_to_le32(bo->hw_res_handle); - virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, &cmd_p->hdr, fence); + vbuf->resp_cb_data = bo; + virtio_gpu_queue_ctrl_buffer(vgdev, vbuf); } void virtio_gpu_cmd_set_scanout(struct virtio_gpu_device *vgdev, @@ -588,10 +600,11 @@ void virtio_gpu_cmd_transfer_to_host_2d(struct virtio_gpu_device *vgdev, struct virtio_gpu_transfer_to_host_2d *cmd_p; struct virtio_gpu_vbuffer *vbuf; bool use_dma_api = !virtio_has_iommu_quirk(vgdev->vdev); + struct virtio_gpu_object_shmem *shmem = to_virtio_gpu_shmem(bo); if (use_dma_api) dma_sync_sg_for_device(vgdev->vdev->dev.parent, - bo->pages->sgl, bo->pages->nents, + shmem->pages->sgl, shmem->pages->nents, DMA_TO_DEVICE); cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p)); @@ -606,7 +619,7 @@ void virtio_gpu_cmd_transfer_to_host_2d(struct virtio_gpu_device *vgdev, cmd_p->r.x = cpu_to_le32(x); cmd_p->r.y = cpu_to_le32(y); - virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, &cmd_p->hdr, fence); + virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence); } static void @@ -629,7 +642,7 @@ virtio_gpu_cmd_resource_attach_backing(struct virtio_gpu_device *vgdev, vbuf->data_buf = ents; vbuf->data_size = sizeof(*ents) * nents; - virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, &cmd_p->hdr, fence); + virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence); } static void virtio_gpu_cmd_get_display_info_cb(struct virtio_gpu_device *vgdev, @@ -939,7 +952,6 @@ void virtio_gpu_cmd_context_attach_resource(struct virtio_gpu_device *vgdev, cmd_p->hdr.ctx_id = cpu_to_le32(ctx_id); cmd_p->resource_id = cpu_to_le32(bo->hw_res_handle); virtio_gpu_queue_ctrl_buffer(vgdev, vbuf); - } void virtio_gpu_cmd_context_detach_resource(struct virtio_gpu_device *vgdev, @@ -988,7 +1000,8 @@ virtio_gpu_cmd_resource_create_3d(struct virtio_gpu_device *vgdev, cmd_p->nr_samples = cpu_to_le32(params->nr_samples); cmd_p->flags = cpu_to_le32(params->flags); - virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, &cmd_p->hdr, fence); + virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence); + bo->created = true; } @@ -1003,10 +1016,11 @@ void virtio_gpu_cmd_transfer_to_host_3d(struct virtio_gpu_device *vgdev, struct virtio_gpu_transfer_host_3d *cmd_p; struct virtio_gpu_vbuffer *vbuf; bool use_dma_api = !virtio_has_iommu_quirk(vgdev->vdev); + struct virtio_gpu_object_shmem *shmem = to_virtio_gpu_shmem(bo); if (use_dma_api) dma_sync_sg_for_device(vgdev->vdev->dev.parent, - bo->pages->sgl, bo->pages->nents, + shmem->pages->sgl, shmem->pages->nents, DMA_TO_DEVICE); cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p)); @@ -1021,7 +1035,7 @@ void virtio_gpu_cmd_transfer_to_host_3d(struct virtio_gpu_device *vgdev, cmd_p->offset = cpu_to_le64(offset); cmd_p->level = cpu_to_le32(level); - virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, &cmd_p->hdr, fence); + virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence); } void virtio_gpu_cmd_transfer_from_host_3d(struct virtio_gpu_device *vgdev, @@ -1047,7 +1061,7 @@ void virtio_gpu_cmd_transfer_from_host_3d(struct virtio_gpu_device *vgdev, cmd_p->offset = cpu_to_le64(offset); cmd_p->level = cpu_to_le32(level); - virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, &cmd_p->hdr, fence); + virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence); } void virtio_gpu_cmd_submit(struct virtio_gpu_device *vgdev, @@ -1070,94 +1084,19 @@ void virtio_gpu_cmd_submit(struct virtio_gpu_device *vgdev, cmd_p->hdr.ctx_id = cpu_to_le32(ctx_id); cmd_p->size = cpu_to_le32(data_size); - virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, &cmd_p->hdr, fence); + virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence); } int virtio_gpu_object_attach(struct virtio_gpu_device *vgdev, struct virtio_gpu_object *obj, - struct virtio_gpu_fence *fence) + struct virtio_gpu_mem_entry *ents, + unsigned int nents) { - bool use_dma_api = !virtio_has_iommu_quirk(vgdev->vdev); - struct virtio_gpu_mem_entry *ents; - struct scatterlist *sg; - int si, nents, ret; - - if (WARN_ON_ONCE(!obj->created)) - return -EINVAL; - if (WARN_ON_ONCE(obj->pages)) - return -EINVAL; - - ret = drm_gem_shmem_pin(&obj->base.base); - if (ret < 0) - return -EINVAL; - - obj->pages = drm_gem_shmem_get_sg_table(&obj->base.base); - if (obj->pages == NULL) { - drm_gem_shmem_unpin(&obj->base.base); - return -EINVAL; - } - - if (use_dma_api) { - obj->mapped = dma_map_sg(vgdev->vdev->dev.parent, - obj->pages->sgl, obj->pages->nents, - DMA_TO_DEVICE); - nents = obj->mapped; - } else { - nents = obj->pages->nents; - } - - /* gets freed when the ring has consumed it */ - ents = kmalloc_array(nents, sizeof(struct virtio_gpu_mem_entry), - GFP_KERNEL); - if (!ents) { - DRM_ERROR("failed to allocate ent list\n"); - return -ENOMEM; - } - - for_each_sg(obj->pages->sgl, sg, nents, si) { - ents[si].addr = cpu_to_le64(use_dma_api - ? sg_dma_address(sg) - : sg_phys(sg)); - ents[si].length = cpu_to_le32(sg->length); - ents[si].padding = 0; - } - virtio_gpu_cmd_resource_attach_backing(vgdev, obj->hw_res_handle, - ents, nents, - fence); + ents, nents, NULL); return 0; } -void virtio_gpu_object_detach(struct virtio_gpu_device *vgdev, - struct virtio_gpu_object *obj) -{ - bool use_dma_api = !virtio_has_iommu_quirk(vgdev->vdev); - - if (WARN_ON_ONCE(!obj->pages)) - return; - - if (use_dma_api && obj->mapped) { - struct virtio_gpu_fence *fence = virtio_gpu_fence_alloc(vgdev); - /* detach backing and wait for the host process it ... */ - virtio_gpu_cmd_resource_inval_backing(vgdev, obj->hw_res_handle, fence); - dma_fence_wait(&fence->f, true); - dma_fence_put(&fence->f); - - /* ... then tear down iommu mappings */ - dma_unmap_sg(vgdev->vdev->dev.parent, - obj->pages->sgl, obj->mapped, - DMA_TO_DEVICE); - obj->mapped = 0; - } else { - virtio_gpu_cmd_resource_inval_backing(vgdev, obj->hw_res_handle, NULL); - } - - sg_free_table(obj->pages); - obj->pages = NULL; - - drm_gem_shmem_unpin(&obj->base.base); -} - void virtio_gpu_cursor_ping(struct virtio_gpu_device *vgdev, struct virtio_gpu_output *output) { |