diff options
Diffstat (limited to 'drivers/virtio')
| -rw-r--r-- | drivers/virtio/virtio.c | 148 | ||||
| -rw-r--r-- | drivers/virtio/virtio_balloon.c | 2 | ||||
| -rw-r--r-- | drivers/virtio/virtio_mem.c | 107 | ||||
| -rw-r--r-- | drivers/virtio/virtio_pci_common.c | 41 | ||||
| -rw-r--r-- | drivers/virtio/virtio_pci_modern.c | 17 | ||||
| -rw-r--r-- | drivers/virtio/virtio_ring.c | 10 |
6 files changed, 286 insertions, 39 deletions
diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index b9095751e43b..95d5d7993e5b 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -377,6 +377,58 @@ static void virtio_dev_remove(struct device *_d) of_node_put(dev->dev.of_node); } +/* + * virtio_irq_get_affinity - get IRQ affinity mask for device + * @_d: ptr to dev structure + * @irq_vec: interrupt vector number + * + * Return the CPU affinity mask for @_d and @irq_vec. + */ +static const struct cpumask *virtio_irq_get_affinity(struct device *_d, + unsigned int irq_vec) +{ + struct virtio_device *dev = dev_to_virtio(_d); + + if (!dev->config->get_vq_affinity) + return NULL; + + return dev->config->get_vq_affinity(dev, irq_vec); +} + +static void virtio_dev_shutdown(struct device *_d) +{ + struct virtio_device *dev = dev_to_virtio(_d); + struct virtio_driver *drv = drv_to_virtio(dev->dev.driver); + + /* + * Stop accesses to or from the device. + * We only need to do it if there's a driver - no accesses otherwise. + */ + if (!drv) + return; + + /* If the driver has its own shutdown method, use that. */ + if (drv->shutdown) { + drv->shutdown(dev); + return; + } + + /* + * Some devices get wedged if you kick them after they are + * reset. Mark all vqs as broken to make sure we don't. + */ + virtio_break_device(dev); + /* + * Guarantee that any callback will see vq->broken as true. + */ + virtio_synchronize_cbs(dev); + /* + * As IOMMUs are reset on shutdown, this will block device access to memory. + * Some devices get wedged if this happens, so reset to make sure it does not. + */ + dev->config->reset(dev); +} + static const struct bus_type virtio_bus = { .name = "virtio", .match = virtio_dev_match, @@ -384,6 +436,8 @@ static const struct bus_type virtio_bus = { .uevent = virtio_uevent, .probe = virtio_dev_probe, .remove = virtio_dev_remove, + .irq_get_affinity = virtio_irq_get_affinity, + .shutdown = virtio_dev_shutdown, }; int __register_virtio_driver(struct virtio_driver *driver, struct module *owner) @@ -527,29 +581,7 @@ void unregister_virtio_device(struct virtio_device *dev) } EXPORT_SYMBOL_GPL(unregister_virtio_device); -#ifdef CONFIG_PM_SLEEP -int virtio_device_freeze(struct virtio_device *dev) -{ - struct virtio_driver *drv = drv_to_virtio(dev->dev.driver); - int ret; - - virtio_config_core_disable(dev); - - dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED; - - if (drv && drv->freeze) { - ret = drv->freeze(dev); - if (ret) { - virtio_config_core_enable(dev); - return ret; - } - } - - return 0; -} -EXPORT_SYMBOL_GPL(virtio_device_freeze); - -int virtio_device_restore(struct virtio_device *dev) +static int virtio_device_restore_priv(struct virtio_device *dev, bool restore) { struct virtio_driver *drv = drv_to_virtio(dev->dev.driver); int ret; @@ -580,8 +612,14 @@ int virtio_device_restore(struct virtio_device *dev) if (ret) goto err; - if (drv->restore) { - ret = drv->restore(dev); + if (restore) { + if (drv->restore) { + ret = drv->restore(dev); + if (ret) + goto err; + } + } else { + ret = drv->reset_done(dev); if (ret) goto err; } @@ -598,9 +636,69 @@ err: virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED); return ret; } + +#ifdef CONFIG_PM_SLEEP +int virtio_device_freeze(struct virtio_device *dev) +{ + struct virtio_driver *drv = drv_to_virtio(dev->dev.driver); + int ret; + + virtio_config_core_disable(dev); + + dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED; + + if (drv && drv->freeze) { + ret = drv->freeze(dev); + if (ret) { + virtio_config_core_enable(dev); + return ret; + } + } + + return 0; +} +EXPORT_SYMBOL_GPL(virtio_device_freeze); + +int virtio_device_restore(struct virtio_device *dev) +{ + return virtio_device_restore_priv(dev, true); +} EXPORT_SYMBOL_GPL(virtio_device_restore); #endif +int virtio_device_reset_prepare(struct virtio_device *dev) +{ + struct virtio_driver *drv = drv_to_virtio(dev->dev.driver); + int ret; + + if (!drv || !drv->reset_prepare) + return -EOPNOTSUPP; + + virtio_config_core_disable(dev); + + dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED; + + ret = drv->reset_prepare(dev); + if (ret) { + virtio_config_core_enable(dev); + return ret; + } + + return 0; +} +EXPORT_SYMBOL_GPL(virtio_device_reset_prepare); + +int virtio_device_reset_done(struct virtio_device *dev) +{ + struct virtio_driver *drv = drv_to_virtio(dev->dev.driver); + + if (!drv || !drv->reset_done) + return -EOPNOTSUPP; + + return virtio_device_restore_priv(dev, false); +} +EXPORT_SYMBOL_GPL(virtio_device_reset_done); + static int virtio_init(void) { if (bus_register(&virtio_bus) != 0) diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index b36d2803674e..89da052f4f68 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -251,7 +251,7 @@ static unsigned int fill_balloon(struct virtio_balloon *vb, size_t num) for (num_pfns = 0; num_pfns < num; num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) { - struct page *page = balloon_page_alloc(); + page = balloon_page_alloc(); if (!page) { dev_info_ratelimited(&vb->vdev->dev, diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index b0b871441578..56d0dbe62163 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -133,6 +133,8 @@ struct virtio_mem { uint64_t addr; /* Maximum region size in bytes. */ uint64_t region_size; + /* Usable region size in bytes. */ + uint64_t usable_region_size; /* The parent resource for all memory added via this device. */ struct resource *parent_resource; @@ -2368,7 +2370,7 @@ static int virtio_mem_cleanup_pending_mb(struct virtio_mem *vm) static void virtio_mem_refresh_config(struct virtio_mem *vm) { const struct range pluggable_range = mhp_get_pluggable_range(true); - uint64_t new_plugged_size, usable_region_size, end_addr; + uint64_t new_plugged_size, end_addr; /* the plugged_size is just a reflection of what _we_ did previously */ virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size, @@ -2378,8 +2380,8 @@ static void virtio_mem_refresh_config(struct virtio_mem *vm) /* calculate the last usable memory block id */ virtio_cread_le(vm->vdev, struct virtio_mem_config, - usable_region_size, &usable_region_size); - end_addr = min(vm->addr + usable_region_size - 1, + usable_region_size, &vm->usable_region_size); + end_addr = min(vm->addr + vm->usable_region_size - 1, pluggable_range.end); if (vm->in_sbm) { @@ -2648,6 +2650,7 @@ static int virtio_mem_init_hotplug(struct virtio_mem *vm) if (rc) goto out_unreg_pm; + virtio_device_ready(vm->vdev); return 0; out_unreg_pm: unregister_pm_notifier(&vm->pm_notifier); @@ -2725,13 +2728,103 @@ static bool virtio_mem_vmcore_pfn_is_ram(struct vmcore_cb *cb, mutex_unlock(&vm->hotplug_mutex); return is_ram; } + +#ifdef CONFIG_PROC_VMCORE_DEVICE_RAM +static int virtio_mem_vmcore_add_device_ram(struct virtio_mem *vm, + struct list_head *list, uint64_t start, uint64_t end) +{ + int rc; + + rc = vmcore_alloc_add_range(list, start, end - start); + if (rc) + dev_err(&vm->vdev->dev, + "Error adding device RAM range: %d\n", rc); + return rc; +} + +static int virtio_mem_vmcore_get_device_ram(struct vmcore_cb *cb, + struct list_head *list) +{ + struct virtio_mem *vm = container_of(cb, struct virtio_mem, + vmcore_cb); + const uint64_t device_start = vm->addr; + const uint64_t device_end = vm->addr + vm->usable_region_size; + uint64_t chunk_size, cur_start, cur_end, plugged_range_start = 0; + LIST_HEAD(tmp_list); + int rc; + + if (!vm->plugged_size) + return 0; + + /* Process memory sections, unless the device block size is bigger. */ + chunk_size = max_t(uint64_t, PFN_PHYS(PAGES_PER_SECTION), + vm->device_block_size); + + mutex_lock(&vm->hotplug_mutex); + + /* + * We process larger chunks and indicate the complete chunk if any + * block in there is plugged. This reduces the number of pfn_is_ram() + * callbacks and mimic what is effectively being done when the old + * kernel would add complete memory sections/blocks to the elfcore hdr. + */ + cur_start = device_start; + for (cur_start = device_start; cur_start < device_end; cur_start = cur_end) { + cur_end = ALIGN_DOWN(cur_start + chunk_size, chunk_size); + cur_end = min_t(uint64_t, cur_end, device_end); + + rc = virtio_mem_send_state_request(vm, cur_start, + cur_end - cur_start); + + if (rc < 0) { + dev_err(&vm->vdev->dev, + "Error querying block states: %d\n", rc); + goto out; + } else if (rc != VIRTIO_MEM_STATE_UNPLUGGED) { + /* Merge ranges with plugged memory. */ + if (!plugged_range_start) + plugged_range_start = cur_start; + continue; + } + + /* Flush any plugged range. */ + if (plugged_range_start) { + rc = virtio_mem_vmcore_add_device_ram(vm, &tmp_list, + plugged_range_start, + cur_start); + if (rc) + goto out; + plugged_range_start = 0; + } + } + + /* Flush any plugged range. */ + if (plugged_range_start) + rc = virtio_mem_vmcore_add_device_ram(vm, &tmp_list, + plugged_range_start, + cur_start); +out: + mutex_unlock(&vm->hotplug_mutex); + if (rc < 0) { + vmcore_free_ranges(&tmp_list); + return rc; + } + list_splice_tail(&tmp_list, list); + return 0; +} +#endif /* CONFIG_PROC_VMCORE_DEVICE_RAM */ #endif /* CONFIG_PROC_VMCORE */ static int virtio_mem_init_kdump(struct virtio_mem *vm) { + /* We must be prepared to receive a callback immediately. */ + virtio_device_ready(vm->vdev); #ifdef CONFIG_PROC_VMCORE dev_info(&vm->vdev->dev, "memory hot(un)plug disabled in kdump kernel\n"); vm->vmcore_cb.pfn_is_ram = virtio_mem_vmcore_pfn_is_ram; +#ifdef CONFIG_PROC_VMCORE_DEVICE_RAM + vm->vmcore_cb.get_device_ram = virtio_mem_vmcore_get_device_ram; +#endif /* CONFIG_PROC_VMCORE_DEVICE_RAM */ register_vmcore_cb(&vm->vmcore_cb); return 0; #else /* CONFIG_PROC_VMCORE */ @@ -2760,6 +2853,8 @@ static int virtio_mem_init(struct virtio_mem *vm) virtio_cread_le(vm->vdev, struct virtio_mem_config, addr, &vm->addr); virtio_cread_le(vm->vdev, struct virtio_mem_config, region_size, &vm->region_size); + virtio_cread_le(vm->vdev, struct virtio_mem_config, usable_region_size, + &vm->usable_region_size); /* Determine the nid for the device based on the lowest address. */ if (vm->nid == NUMA_NO_NODE) @@ -2855,8 +2950,8 @@ static int virtio_mem_probe(struct virtio_device *vdev) mutex_init(&vm->hotplug_mutex); INIT_LIST_HEAD(&vm->next); spin_lock_init(&vm->removal_lock); - hrtimer_init(&vm->retry_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - vm->retry_timer.function = virtio_mem_timer_expired; + hrtimer_setup(&vm->retry_timer, virtio_mem_timer_expired, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS; vm->in_kdump = is_kdump_kernel(); @@ -2870,8 +2965,6 @@ static int virtio_mem_probe(struct virtio_device *vdev) if (rc) goto out_del_vq; - virtio_device_ready(vdev); - /* trigger a config update to start processing the requested_size */ if (!vm->in_kdump) { atomic_set(&vm->config_changed, 1); diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index 88074451dd61..d6d79af44569 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -794,6 +794,46 @@ static int virtio_pci_sriov_configure(struct pci_dev *pci_dev, int num_vfs) return num_vfs; } +static void virtio_pci_reset_prepare(struct pci_dev *pci_dev) +{ + struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev); + int ret = 0; + + ret = virtio_device_reset_prepare(&vp_dev->vdev); + if (ret) { + if (ret != -EOPNOTSUPP) + dev_warn(&pci_dev->dev, "Reset prepare failure: %d", + ret); + return; + } + + if (pci_is_enabled(pci_dev)) + pci_disable_device(pci_dev); +} + +static void virtio_pci_reset_done(struct pci_dev *pci_dev) +{ + struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev); + int ret; + + if (pci_is_enabled(pci_dev)) + return; + + ret = pci_enable_device(pci_dev); + if (!ret) { + pci_set_master(pci_dev); + ret = virtio_device_reset_done(&vp_dev->vdev); + } + + if (ret && ret != -EOPNOTSUPP) + dev_warn(&pci_dev->dev, "Reset done failure: %d", ret); +} + +static const struct pci_error_handlers virtio_pci_err_handler = { + .reset_prepare = virtio_pci_reset_prepare, + .reset_done = virtio_pci_reset_done, +}; + static struct pci_driver virtio_pci_driver = { .name = "virtio-pci", .id_table = virtio_pci_id_table, @@ -803,6 +843,7 @@ static struct pci_driver virtio_pci_driver = { .driver.pm = &virtio_pci_pm_ops, #endif .sriov_configure = virtio_pci_sriov_configure, + .err_handler = &virtio_pci_err_handler, }; struct virtio_device *virtio_pci_vf_get_pf_dev(struct pci_dev *pdev) diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index 5eaade757860..7182f43ed055 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -48,6 +48,7 @@ void vp_modern_avq_done(struct virtqueue *vq) { struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); struct virtio_pci_admin_vq *admin_vq = &vp_dev->admin_vq; + unsigned int status_size = sizeof(struct virtio_admin_cmd_status); struct virtio_admin_cmd *cmd; unsigned long flags; unsigned int len; @@ -56,7 +57,17 @@ void vp_modern_avq_done(struct virtqueue *vq) do { virtqueue_disable_cb(vq); while ((cmd = virtqueue_get_buf(vq, &len))) { - cmd->result_sg_size = len; + /* If the number of bytes written by the device is less + * than the size of struct virtio_admin_cmd_status, the + * remaining status bytes will remain zero-initialized, + * since the buffer was zeroed during allocation. + * In this case, set the size of command_specific_result + * to 0. + */ + if (len < status_size) + cmd->result_sg_size = 0; + else + cmd->result_sg_size = len - status_size; complete(&cmd->completion); } } while (!virtqueue_enable_cb(vq)); @@ -247,7 +258,7 @@ virtio_pci_admin_cmd_dev_parts_objects_enable(struct virtio_device *virtio_dev) sg_init_one(&data_sg, get_data, sizeof(*get_data)); sg_init_one(&result_sg, result, sizeof(*result)); cmd.opcode = cpu_to_le16(VIRTIO_ADMIN_CMD_DEVICE_CAP_GET); - cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SRIOV); + cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SELF); cmd.data_sg = &data_sg; cmd.result_sg = &result_sg; ret = vp_modern_admin_cmd_exec(virtio_dev, &cmd); @@ -305,7 +316,7 @@ static void virtio_pci_admin_cmd_cap_init(struct virtio_device *virtio_dev) sg_init_one(&result_sg, data, sizeof(*data)); cmd.opcode = cpu_to_le16(VIRTIO_ADMIN_CMD_CAP_ID_LIST_QUERY); - cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SRIOV); + cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SELF); cmd.result_sg = &result_sg; ret = vp_modern_admin_cmd_exec(virtio_dev, &cmd); diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index fdd2d2b07b5a..4397392bfef0 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2650,7 +2650,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq) struct vring_virtqueue *vq = to_vvq(_vq); if (vq->event_triggered) - vq->event_triggered = false; + data_race(vq->event_triggered = false); return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) : virtqueue_enable_cb_delayed_split(_vq); @@ -2797,7 +2797,7 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num, void (*recycle_done)(struct virtqueue *vq)) { struct vring_virtqueue *vq = to_vvq(_vq); - int err; + int err, err_reset; if (num > vq->vq.num_max) return -E2BIG; @@ -2819,7 +2819,11 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num, else err = virtqueue_resize_split(_vq, num); - return virtqueue_enable_after_reset(_vq); + err_reset = virtqueue_enable_after_reset(_vq); + if (err_reset) + return err_reset; + + return err; } EXPORT_SYMBOL_GPL(virtqueue_resize); |