diff options
Diffstat (limited to 'drivers/virtio')
| -rw-r--r-- | drivers/virtio/Kconfig | 64 | ||||
| -rw-r--r-- | drivers/virtio/Makefile | 5 | ||||
| -rw-r--r-- | drivers/virtio/virtio.c | 56 | ||||
| -rw-r--r-- | drivers/virtio/virtio_balloon.c | 4 | ||||
| -rw-r--r-- | drivers/virtio/virtio_debug.c | 27 | ||||
| -rw-r--r-- | drivers/virtio/virtio_dma_buf.c | 2 | ||||
| -rw-r--r-- | drivers/virtio/virtio_mem.c | 2 | ||||
| -rw-r--r-- | drivers/virtio/virtio_mmio.c | 52 | ||||
| -rw-r--r-- | drivers/virtio/virtio_pci_modern.c | 27 | ||||
| -rw-r--r-- | drivers/virtio/virtio_pci_modern_dev.c | 69 | ||||
| -rw-r--r-- | drivers/virtio/virtio_ring.c | 14 | ||||
| -rw-r--r-- | drivers/virtio/virtio_rtc_arm.c | 23 | ||||
| -rw-r--r-- | drivers/virtio/virtio_rtc_class.c | 262 | ||||
| -rw-r--r-- | drivers/virtio/virtio_rtc_driver.c | 1407 | ||||
| -rw-r--r-- | drivers/virtio/virtio_rtc_internal.h | 122 | ||||
| -rw-r--r-- | drivers/virtio/virtio_rtc_ptp.c | 347 | ||||
| -rw-r--r-- | drivers/virtio/virtio_vdpa.c | 53 |
17 files changed, 2366 insertions, 170 deletions
diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index 2eb747311bfd..6db5235a7693 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -188,4 +188,68 @@ config VIRTIO_DEBUG If unsure, say N. +config VIRTIO_RTC + tristate "Virtio RTC driver" + depends on VIRTIO + depends on PTP_1588_CLOCK_OPTIONAL + help + This driver provides current time from a Virtio RTC device. The driver + provides the time through one or more clocks. The Virtio RTC PTP + clocks and/or the Real Time Clock driver for Virtio RTC must be + enabled to expose the clocks to userspace. + + To compile this code as a module, choose M here: the module will be + called virtio_rtc. + + If unsure, say M. + +if VIRTIO_RTC + +comment "WARNING: Consider enabling VIRTIO_RTC_PTP and/or VIRTIO_RTC_CLASS." + depends on !VIRTIO_RTC_PTP && !VIRTIO_RTC_CLASS + +comment "Enable PTP_1588_CLOCK in order to enable VIRTIO_RTC_PTP." + depends on PTP_1588_CLOCK=n + +config VIRTIO_RTC_PTP + bool "Virtio RTC PTP clocks" + default y + depends on PTP_1588_CLOCK + help + This exposes any Virtio RTC clocks as PTP Hardware Clocks (PHCs) to + userspace. The PHC sysfs attribute "clock_name" describes the clock + type. + + If unsure, say Y. + +config VIRTIO_RTC_ARM + bool "Virtio RTC cross-timestamping using Arm Generic Timer" + default y + depends on VIRTIO_RTC_PTP && ARM_ARCH_TIMER + help + This enables Virtio RTC cross-timestamping using the Arm Generic Timer. + It only has an effect if the Virtio RTC device also supports this. The + cross-timestamp is available through the PTP clock driver precise + cross-timestamp ioctl (PTP_SYS_OFFSET_PRECISE2 aka + PTP_SYS_OFFSET_PRECISE). + + If unsure, say Y. + +comment "Enable RTC_CLASS in order to enable VIRTIO_RTC_CLASS." + depends on RTC_CLASS=n + +config VIRTIO_RTC_CLASS + bool "Real Time Clock driver for Virtio RTC" + default y + depends on RTC_CLASS + help + This exposes the Virtio RTC UTC-like clock as a Linux Real Time Clock. + It only has an effect if the Virtio RTC device has a UTC-like clock + which smears leap seconds to avoid steps. The Real Time Clock is + read-only, and may support setting an alarm. + + If unsure, say Y. + +endif # VIRTIO_RTC + endif # VIRTIO_MENU diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile index 58b2b0489fc9..eefcfe90d6b8 100644 --- a/drivers/virtio/Makefile +++ b/drivers/virtio/Makefile @@ -14,3 +14,8 @@ obj-$(CONFIG_VIRTIO_VDPA) += virtio_vdpa.o obj-$(CONFIG_VIRTIO_MEM) += virtio_mem.o obj-$(CONFIG_VIRTIO_DMA_SHARED_BUFFER) += virtio_dma_buf.o obj-$(CONFIG_VIRTIO_DEBUG) += virtio_debug.o +obj-$(CONFIG_VIRTIO_RTC) += virtio_rtc.o +virtio_rtc-y := virtio_rtc_driver.o +virtio_rtc-$(CONFIG_VIRTIO_RTC_PTP) += virtio_rtc_ptp.o +virtio_rtc-$(CONFIG_VIRTIO_RTC_ARM) += virtio_rtc_arm.o +virtio_rtc-$(CONFIG_VIRTIO_RTC_CLASS) += virtio_rtc_class.o diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index 150753c3b578..a09eb4d62f82 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -53,7 +53,7 @@ static ssize_t features_show(struct device *_d, /* We actually represent this as a bitstring, as it could be * arbitrary length in future. */ - for (i = 0; i < sizeof(dev->features)*8; i++) + for (i = 0; i < VIRTIO_FEATURES_MAX; i++) len += sysfs_emit_at(buf, len, "%c", __virtio_test_bit(dev, i) ? '1' : '0'); len += sysfs_emit_at(buf, len, "\n"); @@ -147,7 +147,7 @@ EXPORT_SYMBOL_GPL(virtio_config_changed); /** * virtio_config_driver_disable - disable config change reporting by drivers - * @dev: the device to reset + * @dev: the device to disable * * This is only allowed to be called by a driver and disabling can't * be nested. @@ -162,7 +162,7 @@ EXPORT_SYMBOL_GPL(virtio_config_driver_disable); /** * virtio_config_driver_enable - enable config change reporting by drivers - * @dev: the device to reset + * @dev: the device to enable * * This is only allowed to be called by a driver and enabling can't * be nested. @@ -272,22 +272,22 @@ static int virtio_dev_probe(struct device *_d) int err, i; struct virtio_device *dev = dev_to_virtio(_d); struct virtio_driver *drv = drv_to_virtio(dev->dev.driver); - u64 device_features; - u64 driver_features; + u64 device_features[VIRTIO_FEATURES_DWORDS]; + u64 driver_features[VIRTIO_FEATURES_DWORDS]; u64 driver_features_legacy; /* We have a driver! */ virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER); /* Figure out what features the device supports. */ - device_features = dev->config->get_features(dev); + virtio_get_features(dev, device_features); /* Figure out what features the driver supports. */ - driver_features = 0; + virtio_features_zero(driver_features); for (i = 0; i < drv->feature_table_size; i++) { unsigned int f = drv->feature_table[i]; - BUG_ON(f >= 64); - driver_features |= (1ULL << f); + if (!WARN_ON_ONCE(f >= VIRTIO_FEATURES_MAX)) + virtio_features_set_bit(driver_features, f); } /* Some drivers have a separate feature table for virtio v1.0 */ @@ -295,24 +295,29 @@ static int virtio_dev_probe(struct device *_d) driver_features_legacy = 0; for (i = 0; i < drv->feature_table_size_legacy; i++) { unsigned int f = drv->feature_table_legacy[i]; - BUG_ON(f >= 64); - driver_features_legacy |= (1ULL << f); + if (!WARN_ON_ONCE(f >= 64)) + driver_features_legacy |= (1ULL << f); } } else { - driver_features_legacy = driver_features; + driver_features_legacy = driver_features[0]; } - if (device_features & (1ULL << VIRTIO_F_VERSION_1)) - dev->features = driver_features & device_features; - else - dev->features = driver_features_legacy & device_features; + if (virtio_features_test_bit(device_features, VIRTIO_F_VERSION_1)) { + for (i = 0; i < VIRTIO_FEATURES_DWORDS; ++i) + dev->features_array[i] = driver_features[i] & + device_features[i]; + } else { + virtio_features_from_u64(dev->features_array, + driver_features_legacy & + device_features[0]); + } /* When debugging, user may filter some features by hand. */ virtio_debug_device_filter_features(dev); /* Transport features always preserved to pass to finalize_features. */ for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) - if (device_features & (1ULL << i)) + if (virtio_features_test_bit(device_features, i)) __virtio_set_bit(dev, i); err = dev->config->finalize_features(dev); @@ -320,14 +325,15 @@ static int virtio_dev_probe(struct device *_d) goto err; if (drv->validate) { - u64 features = dev->features; + u64 features[VIRTIO_FEATURES_DWORDS]; + virtio_features_copy(features, dev->features_array); err = drv->validate(dev); if (err) goto err; /* Did validation change any features? Then write them again. */ - if (features != dev->features) { + if (!virtio_features_equal(features, dev->features_array)) { err = dev->config->finalize_features(dev); if (err) goto err; @@ -407,6 +413,12 @@ static void virtio_dev_shutdown(struct device *_d) if (!drv) return; + /* If the driver has its own shutdown method, use that. */ + if (drv->shutdown) { + drv->shutdown(dev); + return; + } + /* * Some devices get wedged if you kick them after they are * reset. Mark all vqs as broken to make sure we don't. @@ -500,7 +512,7 @@ out: * On error, the caller must call put_device on &@dev->dev (and not kfree), * as another code path may have obtained a reference to @dev. * - * Returns: 0 on suceess, -error on failure + * Returns: 0 on success, -error on failure */ int register_virtio_device(struct virtio_device *dev) { @@ -524,6 +536,7 @@ int register_virtio_device(struct virtio_device *dev) goto out_ida_remove; spin_lock_init(&dev->config_lock); + dev->config_driver_disabled = false; dev->config_core_enabled = false; dev->config_change_pending = false; @@ -695,6 +708,9 @@ EXPORT_SYMBOL_GPL(virtio_device_reset_done); static int virtio_init(void) { + BUILD_BUG_ON(offsetof(struct virtio_device, features) != + offsetof(struct virtio_device, features_array[0])); + if (bus_register(&virtio_bus) != 0) panic("virtio bus registration failed"); virtio_debug_init(); diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 89da052f4f68..e299e18346a3 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -866,15 +866,13 @@ static int virtballoon_migratepage(struct balloon_dev_info *vb_dev_info, tell_host(vb, vb->inflate_vq); /* balloon's page migration 2nd step -- deflate "page" */ - spin_lock_irqsave(&vb_dev_info->pages_lock, flags); - balloon_page_delete(page); - spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags); vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE; set_page_pfns(vb, vb->pfns, page); tell_host(vb, vb->deflate_vq); mutex_unlock(&vb->balloon_lock); + balloon_page_finalize(page); put_page(page); /* balloon reference */ return MIGRATEPAGE_SUCCESS; diff --git a/drivers/virtio/virtio_debug.c b/drivers/virtio/virtio_debug.c index 95c8fc7705bb..d58713ddf2e5 100644 --- a/drivers/virtio/virtio_debug.c +++ b/drivers/virtio/virtio_debug.c @@ -8,13 +8,13 @@ static struct dentry *virtio_debugfs_dir; static int virtio_debug_device_features_show(struct seq_file *s, void *data) { + u64 device_features[VIRTIO_FEATURES_DWORDS]; struct virtio_device *dev = s->private; - u64 device_features; unsigned int i; - device_features = dev->config->get_features(dev); - for (i = 0; i < BITS_PER_LONG_LONG; i++) { - if (device_features & (1ULL << i)) + virtio_get_features(dev, device_features); + for (i = 0; i < VIRTIO_FEATURES_MAX; i++) { + if (virtio_features_test_bit(device_features, i)) seq_printf(s, "%u\n", i); } return 0; @@ -26,8 +26,8 @@ static int virtio_debug_filter_features_show(struct seq_file *s, void *data) struct virtio_device *dev = s->private; unsigned int i; - for (i = 0; i < BITS_PER_LONG_LONG; i++) { - if (dev->debugfs_filter_features & (1ULL << i)) + for (i = 0; i < VIRTIO_FEATURES_MAX; i++) { + if (virtio_features_test_bit(dev->debugfs_filter_features, i)) seq_printf(s, "%u\n", i); } return 0; @@ -39,7 +39,7 @@ static int virtio_debug_filter_features_clear(void *data, u64 val) struct virtio_device *dev = data; if (val == 1) - dev->debugfs_filter_features = 0; + virtio_features_zero(dev->debugfs_filter_features); return 0; } @@ -50,9 +50,10 @@ static int virtio_debug_filter_feature_add(void *data, u64 val) { struct virtio_device *dev = data; - if (val >= BITS_PER_LONG_LONG) + if (val >= VIRTIO_FEATURES_MAX) return -EINVAL; - dev->debugfs_filter_features |= BIT_ULL_MASK(val); + + virtio_features_set_bit(dev->debugfs_filter_features, val); return 0; } @@ -63,9 +64,10 @@ static int virtio_debug_filter_feature_del(void *data, u64 val) { struct virtio_device *dev = data; - if (val >= BITS_PER_LONG_LONG) + if (val >= VIRTIO_FEATURES_MAX) return -EINVAL; - dev->debugfs_filter_features &= ~BIT_ULL_MASK(val); + + virtio_features_clear_bit(dev->debugfs_filter_features, val); return 0; } @@ -91,7 +93,8 @@ EXPORT_SYMBOL_GPL(virtio_debug_device_init); void virtio_debug_device_filter_features(struct virtio_device *dev) { - dev->features &= ~dev->debugfs_filter_features; + virtio_features_andnot(dev->features_array, dev->features_array, + dev->debugfs_filter_features); } EXPORT_SYMBOL_GPL(virtio_debug_device_filter_features); diff --git a/drivers/virtio/virtio_dma_buf.c b/drivers/virtio/virtio_dma_buf.c index 3fe1d03b0645..95c10632f84a 100644 --- a/drivers/virtio/virtio_dma_buf.c +++ b/drivers/virtio/virtio_dma_buf.c @@ -36,6 +36,8 @@ EXPORT_SYMBOL(virtio_dma_buf_export); /** * virtio_dma_buf_attach - mandatory attach callback for virtio dma-bufs + * @dma_buf: [in] buffer to attach + * @attach: [in] attachment structure */ int virtio_dma_buf_attach(struct dma_buf *dma_buf, struct dma_buf_attachment *attach) diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index 56d0dbe62163..1688ecd69a04 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -1243,7 +1243,7 @@ static int virtio_mem_fake_offline(struct virtio_mem *vm, unsigned long pfn, if (atomic_read(&vm->config_changed)) return -EAGAIN; - rc = alloc_contig_range(pfn, pfn + nr_pages, MIGRATE_MOVABLE, + rc = alloc_contig_range(pfn, pfn + nr_pages, ACR_FLAGS_NONE, GFP_KERNEL); if (rc == -ENOMEM) /* whoops, out of memory */ diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 5d78c2d572ab..b152a1eca05a 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -65,7 +65,6 @@ #include <linux/platform_device.h> #include <linux/pm.h> #include <linux/slab.h> -#include <linux/spinlock.h> #include <linux/virtio.h> #include <linux/virtio_config.h> #include <uapi/linux/virtio_mmio.h> @@ -88,22 +87,8 @@ struct virtio_mmio_device { void __iomem *base; unsigned long version; - - /* a list of queues so we can dispatch IRQs */ - spinlock_t lock; - struct list_head virtqueues; -}; - -struct virtio_mmio_vq_info { - /* the actual virtqueue */ - struct virtqueue *vq; - - /* the list node for the virtqueues list */ - struct list_head node; }; - - /* Configuration interface */ static u64 vm_get_features(struct virtio_device *vdev) @@ -300,9 +285,8 @@ static bool vm_notify_with_data(struct virtqueue *vq) static irqreturn_t vm_interrupt(int irq, void *opaque) { struct virtio_mmio_device *vm_dev = opaque; - struct virtio_mmio_vq_info *info; + struct virtqueue *vq; unsigned long status; - unsigned long flags; irqreturn_t ret = IRQ_NONE; /* Read and acknowledge interrupts */ @@ -315,10 +299,8 @@ static irqreturn_t vm_interrupt(int irq, void *opaque) } if (likely(status & VIRTIO_MMIO_INT_VRING)) { - spin_lock_irqsave(&vm_dev->lock, flags); - list_for_each_entry(info, &vm_dev->virtqueues, node) - ret |= vring_interrupt(irq, info->vq); - spin_unlock_irqrestore(&vm_dev->lock, flags); + virtio_device_for_each_vq(&vm_dev->vdev, vq) + ret |= vring_interrupt(irq, vq); } return ret; @@ -329,14 +311,8 @@ static irqreturn_t vm_interrupt(int irq, void *opaque) static void vm_del_vq(struct virtqueue *vq) { struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vq->vdev); - struct virtio_mmio_vq_info *info = vq->priv; - unsigned long flags; unsigned int index = vq->index; - spin_lock_irqsave(&vm_dev->lock, flags); - list_del(&info->node); - spin_unlock_irqrestore(&vm_dev->lock, flags); - /* Select and deactivate the queue */ writel(index, vm_dev->base + VIRTIO_MMIO_QUEUE_SEL); if (vm_dev->version == 1) { @@ -347,8 +323,6 @@ static void vm_del_vq(struct virtqueue *vq) } vring_del_virtqueue(vq); - - kfree(info); } static void vm_del_vqs(struct virtio_device *vdev) @@ -375,9 +349,7 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in { struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); bool (*notify)(struct virtqueue *vq); - struct virtio_mmio_vq_info *info; struct virtqueue *vq; - unsigned long flags; unsigned int num; int err; @@ -399,13 +371,6 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in goto error_available; } - /* Allocate and fill out our active queue description */ - info = kmalloc(sizeof(*info), GFP_KERNEL); - if (!info) { - err = -ENOMEM; - goto error_kmalloc; - } - num = readl(vm_dev->base + VIRTIO_MMIO_QUEUE_NUM_MAX); if (num == 0) { err = -ENOENT; @@ -463,13 +428,6 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in writel(1, vm_dev->base + VIRTIO_MMIO_QUEUE_READY); } - vq->priv = info; - info->vq = vq; - - spin_lock_irqsave(&vm_dev->lock, flags); - list_add(&info->node, &vm_dev->virtqueues); - spin_unlock_irqrestore(&vm_dev->lock, flags); - return vq; error_bad_pfn: @@ -481,8 +439,6 @@ error_new_virtqueue: writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_READY); WARN_ON(readl(vm_dev->base + VIRTIO_MMIO_QUEUE_READY)); } - kfree(info); -error_kmalloc: error_available: return ERR_PTR(err); } @@ -627,8 +583,6 @@ static int virtio_mmio_probe(struct platform_device *pdev) vm_dev->vdev.dev.release = virtio_mmio_release_dev; vm_dev->vdev.config = &virtio_mmio_config_ops; vm_dev->pdev = pdev; - INIT_LIST_HEAD(&vm_dev->virtqueues); - spin_lock_init(&vm_dev->lock); vm_dev->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(vm_dev->base)) { diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index 5eaade757860..dd0e65f71d41 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -22,11 +22,11 @@ #define VIRTIO_AVQ_SGS_MAX 4 -static u64 vp_get_features(struct virtio_device *vdev) +static void vp_get_features(struct virtio_device *vdev, u64 *features) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); - return vp_modern_get_features(&vp_dev->mdev); + vp_modern_get_extended_features(&vp_dev->mdev, features); } static int vp_avq_index(struct virtio_device *vdev, u16 *index, u16 *num) @@ -48,6 +48,7 @@ void vp_modern_avq_done(struct virtqueue *vq) { struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); struct virtio_pci_admin_vq *admin_vq = &vp_dev->admin_vq; + unsigned int status_size = sizeof(struct virtio_admin_cmd_status); struct virtio_admin_cmd *cmd; unsigned long flags; unsigned int len; @@ -56,7 +57,17 @@ void vp_modern_avq_done(struct virtqueue *vq) do { virtqueue_disable_cb(vq); while ((cmd = virtqueue_get_buf(vq, &len))) { - cmd->result_sg_size = len; + /* If the number of bytes written by the device is less + * than the size of struct virtio_admin_cmd_status, the + * remaining status bytes will remain zero-initialized, + * since the buffer was zeroed during allocation. + * In this case, set the size of command_specific_result + * to 0. + */ + if (len < status_size) + cmd->result_sg_size = 0; + else + cmd->result_sg_size = len - status_size; complete(&cmd->completion); } } while (!virtqueue_enable_cb(vq)); @@ -247,7 +258,7 @@ virtio_pci_admin_cmd_dev_parts_objects_enable(struct virtio_device *virtio_dev) sg_init_one(&data_sg, get_data, sizeof(*get_data)); sg_init_one(&result_sg, result, sizeof(*result)); cmd.opcode = cpu_to_le16(VIRTIO_ADMIN_CMD_DEVICE_CAP_GET); - cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SRIOV); + cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SELF); cmd.data_sg = &data_sg; cmd.result_sg = &result_sg; ret = vp_modern_admin_cmd_exec(virtio_dev, &cmd); @@ -305,7 +316,7 @@ static void virtio_pci_admin_cmd_cap_init(struct virtio_device *virtio_dev) sg_init_one(&result_sg, data, sizeof(*data)); cmd.opcode = cpu_to_le16(VIRTIO_ADMIN_CMD_CAP_ID_LIST_QUERY); - cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SRIOV); + cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SELF); cmd.result_sg = &result_sg; ret = vp_modern_admin_cmd_exec(virtio_dev, &cmd); @@ -426,7 +437,7 @@ static int vp_finalize_features(struct virtio_device *vdev) if (vp_check_common_size(vdev)) return -EINVAL; - vp_modern_set_features(&vp_dev->mdev, vdev->features); + vp_modern_set_extended_features(&vp_dev->mdev, vdev->features_array); return 0; } @@ -1223,7 +1234,7 @@ static const struct virtio_config_ops virtio_pci_config_nodev_ops = { .find_vqs = vp_modern_find_vqs, .del_vqs = vp_del_vqs, .synchronize_cbs = vp_synchronize_vectors, - .get_features = vp_get_features, + .get_extended_features = vp_get_features, .finalize_features = vp_finalize_features, .bus_name = vp_bus_name, .set_vq_affinity = vp_set_vq_affinity, @@ -1243,7 +1254,7 @@ static const struct virtio_config_ops virtio_pci_config_ops = { .find_vqs = vp_modern_find_vqs, .del_vqs = vp_del_vqs, .synchronize_cbs = vp_synchronize_vectors, - .get_features = vp_get_features, + .get_extended_features = vp_get_features, .finalize_features = vp_finalize_features, .bus_name = vp_bus_name, .set_vq_affinity = vp_set_vq_affinity, diff --git a/drivers/virtio/virtio_pci_modern_dev.c b/drivers/virtio/virtio_pci_modern_dev.c index 0d3dbfaf4b23..d665f8f73ea8 100644 --- a/drivers/virtio/virtio_pci_modern_dev.c +++ b/drivers/virtio/virtio_pci_modern_dev.c @@ -388,63 +388,74 @@ void vp_modern_remove(struct virtio_pci_modern_device *mdev) EXPORT_SYMBOL_GPL(vp_modern_remove); /* - * vp_modern_get_features - get features from device + * vp_modern_get_extended_features - get features from device * @mdev: the modern virtio-pci device + * @features: the features array to be filled * - * Returns the features read from the device + * Fill the specified features array with the features read from the device */ -u64 vp_modern_get_features(struct virtio_pci_modern_device *mdev) +void vp_modern_get_extended_features(struct virtio_pci_modern_device *mdev, + u64 *features) { struct virtio_pci_common_cfg __iomem *cfg = mdev->common; + int i; - u64 features; + virtio_features_zero(features); + for (i = 0; i < VIRTIO_FEATURES_WORDS; i++) { + u64 cur; - vp_iowrite32(0, &cfg->device_feature_select); - features = vp_ioread32(&cfg->device_feature); - vp_iowrite32(1, &cfg->device_feature_select); - features |= ((u64)vp_ioread32(&cfg->device_feature) << 32); - - return features; + vp_iowrite32(i, &cfg->device_feature_select); + cur = vp_ioread32(&cfg->device_feature); + features[i >> 1] |= cur << (32 * (i & 1)); + } } -EXPORT_SYMBOL_GPL(vp_modern_get_features); +EXPORT_SYMBOL_GPL(vp_modern_get_extended_features); /* * vp_modern_get_driver_features - get driver features from device * @mdev: the modern virtio-pci device + * @features: the features array to be filled * - * Returns the driver features read from the device + * Fill the specified features array with the driver features read from the + * device */ -u64 vp_modern_get_driver_features(struct virtio_pci_modern_device *mdev) +void +vp_modern_get_driver_extended_features(struct virtio_pci_modern_device *mdev, + u64 *features) { struct virtio_pci_common_cfg __iomem *cfg = mdev->common; + int i; - u64 features; - - vp_iowrite32(0, &cfg->guest_feature_select); - features = vp_ioread32(&cfg->guest_feature); - vp_iowrite32(1, &cfg->guest_feature_select); - features |= ((u64)vp_ioread32(&cfg->guest_feature) << 32); + virtio_features_zero(features); + for (i = 0; i < VIRTIO_FEATURES_WORDS; i++) { + u64 cur; - return features; + vp_iowrite32(i, &cfg->guest_feature_select); + cur = vp_ioread32(&cfg->guest_feature); + features[i >> 1] |= cur << (32 * (i & 1)); + } } -EXPORT_SYMBOL_GPL(vp_modern_get_driver_features); +EXPORT_SYMBOL_GPL(vp_modern_get_driver_extended_features); /* - * vp_modern_set_features - set features to device + * vp_modern_set_extended_features - set features to device * @mdev: the modern virtio-pci device * @features: the features set to device */ -void vp_modern_set_features(struct virtio_pci_modern_device *mdev, - u64 features) +void vp_modern_set_extended_features(struct virtio_pci_modern_device *mdev, + const u64 *features) { struct virtio_pci_common_cfg __iomem *cfg = mdev->common; + int i; + + for (i = 0; i < VIRTIO_FEATURES_WORDS; i++) { + u32 cur = features[i >> 1] >> (32 * (i & 1)); - vp_iowrite32(0, &cfg->guest_feature_select); - vp_iowrite32((u32)features, &cfg->guest_feature); - vp_iowrite32(1, &cfg->guest_feature_select); - vp_iowrite32(features >> 32, &cfg->guest_feature); + vp_iowrite32(i, &cfg->guest_feature_select); + vp_iowrite32(cur, &cfg->guest_feature); + } } -EXPORT_SYMBOL_GPL(vp_modern_set_features); +EXPORT_SYMBOL_GPL(vp_modern_set_extended_features); /* * vp_modern_generation - get the device genreation diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index fdd2d2b07b5a..f5062061c408 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2296,6 +2296,10 @@ static inline int virtqueue_add(struct virtqueue *_vq, * at the same time (except where noted). * * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO). + * + * NB: ENOSPC is a special code that is only returned on an attempt to add a + * buffer to a full VQ. It indicates that some buffers are outstanding and that + * the operation can be retried after some buffers have been used. */ int virtqueue_add_sgs(struct virtqueue *_vq, struct scatterlist *sgs[], @@ -2650,7 +2654,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq) struct vring_virtqueue *vq = to_vvq(_vq); if (vq->event_triggered) - vq->event_triggered = false; + data_race(vq->event_triggered = false); return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) : virtqueue_enable_cb_delayed_split(_vq); @@ -2797,7 +2801,7 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num, void (*recycle_done)(struct virtqueue *vq)) { struct vring_virtqueue *vq = to_vvq(_vq); - int err; + int err, err_reset; if (num > vq->vq.num_max) return -E2BIG; @@ -2819,7 +2823,11 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num, else err = virtqueue_resize_split(_vq, num); - return virtqueue_enable_after_reset(_vq); + err_reset = virtqueue_enable_after_reset(_vq); + if (err_reset) + return err_reset; + + return err; } EXPORT_SYMBOL_GPL(virtqueue_resize); diff --git a/drivers/virtio/virtio_rtc_arm.c b/drivers/virtio/virtio_rtc_arm.c new file mode 100644 index 000000000000..211299d72870 --- /dev/null +++ b/drivers/virtio/virtio_rtc_arm.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Provides cross-timestamp params for Arm. + * + * Copyright (C) 2022-2023 OpenSynergy GmbH + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include <linux/clocksource_ids.h> + +#include <uapi/linux/virtio_rtc.h> + +#include "virtio_rtc_internal.h" + +/* see header for doc */ + +int viortc_hw_xtstamp_params(u8 *hw_counter, enum clocksource_ids *cs_id) +{ + *hw_counter = VIRTIO_RTC_COUNTER_ARM_VCT; + *cs_id = CSID_ARM_ARCH_COUNTER; + + return 0; +} diff --git a/drivers/virtio/virtio_rtc_class.c b/drivers/virtio/virtio_rtc_class.c new file mode 100644 index 000000000000..05d6d28255cf --- /dev/null +++ b/drivers/virtio/virtio_rtc_class.c @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * virtio_rtc RTC class driver + * + * Copyright (C) 2023 OpenSynergy GmbH + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include <linux/math64.h> +#include <linux/overflow.h> +#include <linux/rtc.h> +#include <linux/time64.h> + +#include <uapi/linux/virtio_rtc.h> + +#include "virtio_rtc_internal.h" + +/** + * struct viortc_class - RTC class wrapper + * @viortc: virtio_rtc device data + * @rtc: RTC device + * @vio_clk_id: virtio_rtc clock id + * @stopped: Whether RTC ops are disallowed. Access protected by rtc_lock(). + */ +struct viortc_class { + struct viortc_dev *viortc; + struct rtc_device *rtc; + u16 vio_clk_id; + bool stopped; +}; + +/** + * viortc_class_get_locked() - get RTC class wrapper, if ops allowed + * @dev: virtio device + * + * Gets the RTC class wrapper from the virtio device, if it is available and + * ops are allowed. + * + * Context: Caller must hold rtc_lock(). + * Return: RTC class wrapper if available and ops allowed, ERR_PTR otherwise. + */ +static struct viortc_class *viortc_class_get_locked(struct device *dev) +{ + struct viortc_class *viortc_class; + + viortc_class = viortc_class_from_dev(dev); + if (IS_ERR(viortc_class)) + return viortc_class; + + if (viortc_class->stopped) + return ERR_PTR(-EBUSY); + + return viortc_class; +} + +/** + * viortc_class_read_time() - RTC class op read_time + * @dev: virtio device + * @tm: read time + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_class_read_time(struct device *dev, struct rtc_time *tm) +{ + struct viortc_class *viortc_class; + time64_t sec; + int ret; + u64 ns; + + viortc_class = viortc_class_get_locked(dev); + if (IS_ERR(viortc_class)) + return PTR_ERR(viortc_class); + + ret = viortc_read(viortc_class->viortc, viortc_class->vio_clk_id, &ns); + if (ret) + return ret; + + sec = div_u64(ns, NSEC_PER_SEC); + + rtc_time64_to_tm(sec, tm); + + return 0; +} + +/** + * viortc_class_read_alarm() - RTC class op read_alarm + * @dev: virtio device + * @alrm: alarm read out + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_class_read_alarm(struct device *dev, struct rtc_wkalrm *alrm) +{ + struct viortc_class *viortc_class; + time64_t alarm_time_sec; + u64 alarm_time_ns; + bool enabled; + int ret; + + viortc_class = viortc_class_get_locked(dev); + if (IS_ERR(viortc_class)) + return PTR_ERR(viortc_class); + + ret = viortc_read_alarm(viortc_class->viortc, viortc_class->vio_clk_id, + &alarm_time_ns, &enabled); + if (ret) + return ret; + + alarm_time_sec = div_u64(alarm_time_ns, NSEC_PER_SEC); + rtc_time64_to_tm(alarm_time_sec, &alrm->time); + + alrm->enabled = enabled; + + return 0; +} + +/** + * viortc_class_set_alarm() - RTC class op set_alarm + * @dev: virtio device + * @alrm: alarm to set + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_class_set_alarm(struct device *dev, struct rtc_wkalrm *alrm) +{ + struct viortc_class *viortc_class; + time64_t alarm_time_sec; + u64 alarm_time_ns; + + viortc_class = viortc_class_get_locked(dev); + if (IS_ERR(viortc_class)) + return PTR_ERR(viortc_class); + + alarm_time_sec = rtc_tm_to_time64(&alrm->time); + + if (alarm_time_sec < 0) + return -EINVAL; + + if (check_mul_overflow((u64)alarm_time_sec, (u64)NSEC_PER_SEC, + &alarm_time_ns)) + return -EINVAL; + + return viortc_set_alarm(viortc_class->viortc, viortc_class->vio_clk_id, + alarm_time_ns, alrm->enabled); +} + +/** + * viortc_class_alarm_irq_enable() - RTC class op alarm_irq_enable + * @dev: virtio device + * @enabled: enable or disable alarm IRQ + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_class_alarm_irq_enable(struct device *dev, + unsigned int enabled) +{ + struct viortc_class *viortc_class; + + viortc_class = viortc_class_get_locked(dev); + if (IS_ERR(viortc_class)) + return PTR_ERR(viortc_class); + + return viortc_set_alarm_enabled(viortc_class->viortc, + viortc_class->vio_clk_id, enabled); +} + +static const struct rtc_class_ops viortc_class_ops = { + .read_time = viortc_class_read_time, + .read_alarm = viortc_class_read_alarm, + .set_alarm = viortc_class_set_alarm, + .alarm_irq_enable = viortc_class_alarm_irq_enable, +}; + +/** + * viortc_class_alarm() - propagate alarm notification as alarm interrupt + * @viortc_class: RTC class wrapper + * @vio_clk_id: virtio_rtc clock id + * + * Context: Any context. + */ +void viortc_class_alarm(struct viortc_class *viortc_class, u16 vio_clk_id) +{ + if (vio_clk_id != viortc_class->vio_clk_id) { + dev_warn_ratelimited(&viortc_class->rtc->dev, + "ignoring alarm for clock id %d, expected id %d\n", + vio_clk_id, viortc_class->vio_clk_id); + return; + } + + rtc_update_irq(viortc_class->rtc, 1, RTC_AF | RTC_IRQF); +} + +/** + * viortc_class_stop() - disallow RTC class ops + * @viortc_class: RTC class wrapper + * + * Context: Process context. Caller must NOT hold rtc_lock(). + */ +void viortc_class_stop(struct viortc_class *viortc_class) +{ + rtc_lock(viortc_class->rtc); + + viortc_class->stopped = true; + + rtc_unlock(viortc_class->rtc); +} + +/** + * viortc_class_register() - register RTC class device + * @viortc_class: RTC class wrapper + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +int viortc_class_register(struct viortc_class *viortc_class) +{ + return devm_rtc_register_device(viortc_class->rtc); +} + +/** + * viortc_class_init() - init RTC class wrapper and device + * @viortc: device data + * @vio_clk_id: virtio_rtc clock id + * @have_alarm: have alarm feature + * @parent_dev: virtio device + * + * Context: Process context. + * Return: RTC class wrapper on success, ERR_PTR otherwise. + */ +struct viortc_class *viortc_class_init(struct viortc_dev *viortc, + u16 vio_clk_id, bool have_alarm, + struct device *parent_dev) +{ + struct viortc_class *viortc_class; + struct rtc_device *rtc; + + viortc_class = + devm_kzalloc(parent_dev, sizeof(*viortc_class), GFP_KERNEL); + if (!viortc_class) + return ERR_PTR(-ENOMEM); + + rtc = devm_rtc_allocate_device(parent_dev); + if (IS_ERR(rtc)) + return ERR_CAST(rtc); + + viortc_class->viortc = viortc; + viortc_class->rtc = rtc; + viortc_class->vio_clk_id = vio_clk_id; + + if (!have_alarm) + clear_bit(RTC_FEATURE_ALARM, rtc->features); + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, rtc->features); + + rtc->ops = &viortc_class_ops; + rtc->range_max = div_u64(U64_MAX, NSEC_PER_SEC); + + return viortc_class; +} diff --git a/drivers/virtio/virtio_rtc_driver.c b/drivers/virtio/virtio_rtc_driver.c new file mode 100644 index 000000000000..a57d5e06e19d --- /dev/null +++ b/drivers/virtio/virtio_rtc_driver.c @@ -0,0 +1,1407 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * virtio_rtc driver core + * + * Copyright (C) 2022-2024 OpenSynergy GmbH + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include <linux/completion.h> +#include <linux/device.h> +#include <linux/module.h> +#include <linux/pm.h> +#include <linux/virtio.h> +#include <linux/virtio_config.h> +#include <linux/virtio_ids.h> + +#include <uapi/linux/virtio_rtc.h> + +#include "virtio_rtc_internal.h" + +#define VIORTC_ALARMQ_BUF_CAP sizeof(union virtio_rtc_notif_alarmq) + +/* virtqueue order */ +enum { + VIORTC_REQUESTQ, + VIORTC_ALARMQ, + VIORTC_MAX_NR_QUEUES, +}; + +/** + * struct viortc_vq - virtqueue abstraction + * @vq: virtqueue + * @lock: protects access to vq + */ +struct viortc_vq { + struct virtqueue *vq; + spinlock_t lock; +}; + +/** + * struct viortc_dev - virtio_rtc device data + * @vdev: virtio device + * @viortc_class: RTC class wrapper for UTC-like clock, NULL if not available + * @vqs: virtqueues + * @clocks_to_unregister: Clock references, which are only used during device + * removal. + * For other uses, there would be a race between device + * creation and setting the pointers here. + * @alarmq_bufs: alarmq buffers list + * @num_alarmq_bufs: # of alarmq buffers + * @num_clocks: # of virtio_rtc clocks + */ +struct viortc_dev { + struct virtio_device *vdev; + struct viortc_class *viortc_class; + struct viortc_vq vqs[VIORTC_MAX_NR_QUEUES]; + struct viortc_ptp_clock **clocks_to_unregister; + void **alarmq_bufs; + unsigned int num_alarmq_bufs; + u16 num_clocks; +}; + +/** + * struct viortc_msg - Message requested by driver, responded by device. + * @viortc: device data + * @req: request buffer + * @resp: response buffer + * @responded: vqueue callback signals response reception + * @refcnt: Message reference count, message and buffers will be deallocated + * once 0. refcnt is decremented in the vqueue callback and in the + * thread waiting on the responded completion. + * If a message response wait function times out, the message will be + * freed upon late reception (refcnt will reach 0 in the callback), or + * device removal. + * @req_size: size of request in bytes + * @resp_cap: maximum size of response in bytes + * @resp_actual_size: actual size of response + */ +struct viortc_msg { + struct viortc_dev *viortc; + void *req; + void *resp; + struct completion responded; + refcount_t refcnt; + unsigned int req_size; + unsigned int resp_cap; + unsigned int resp_actual_size; +}; + +/** + * viortc_class_from_dev() - Get RTC class object from virtio device. + * @dev: virtio device + * + * Context: Any context. + * Return: RTC class object if available, ERR_PTR otherwise. + */ +struct viortc_class *viortc_class_from_dev(struct device *dev) +{ + struct virtio_device *vdev; + struct viortc_dev *viortc; + + vdev = container_of(dev, typeof(*vdev), dev); + viortc = vdev->priv; + + return viortc->viortc_class ?: ERR_PTR(-ENODEV); +} + +/** + * viortc_alarms_supported() - Whether device and driver support alarms. + * @vdev: virtio device + * + * NB: Device and driver may not support alarms for the same clocks. + * + * Context: Any context. + * Return: True if both device and driver can support alarms. + */ +static bool viortc_alarms_supported(struct virtio_device *vdev) +{ + return IS_ENABLED(CONFIG_VIRTIO_RTC_CLASS) && + virtio_has_feature(vdev, VIRTIO_RTC_F_ALARM); +} + +/** + * viortc_feed_vq() - Make a device write-only buffer available. + * @viortc: device data + * @vq: notification virtqueue + * @buf: buffer + * @buf_len: buffer capacity in bytes + * @data: token, identifying buffer + * + * Context: Caller must prevent concurrent access to vq. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_feed_vq(struct viortc_dev *viortc, struct virtqueue *vq, + void *buf, unsigned int buf_len, void *data) +{ + struct scatterlist sg; + + sg_init_one(&sg, buf, buf_len); + + return virtqueue_add_inbuf(vq, &sg, 1, data, GFP_ATOMIC); +} + +/** + * viortc_msg_init() - Allocate and initialize requestq message. + * @viortc: device data + * @msg_type: virtio_rtc message type + * @req_size: size of request buffer to be allocated + * @resp_cap: size of response buffer to be allocated + * + * Initializes the message refcnt to 2. The refcnt will be decremented once in + * the virtqueue callback, and once in the thread waiting on the message (on + * completion or timeout). + * + * Context: Process context. + * Return: non-NULL on success. + */ +static struct viortc_msg *viortc_msg_init(struct viortc_dev *viortc, + u16 msg_type, unsigned int req_size, + unsigned int resp_cap) +{ + struct device *dev = &viortc->vdev->dev; + struct virtio_rtc_req_head *req_head; + struct viortc_msg *msg; + + msg = devm_kzalloc(dev, sizeof(*msg), GFP_KERNEL); + if (!msg) + return NULL; + + init_completion(&msg->responded); + + msg->req = devm_kzalloc(dev, req_size, GFP_KERNEL); + if (!msg->req) + goto err_free_msg; + + req_head = msg->req; + + msg->resp = devm_kzalloc(dev, resp_cap, GFP_KERNEL); + if (!msg->resp) + goto err_free_msg_req; + + msg->viortc = viortc; + msg->req_size = req_size; + msg->resp_cap = resp_cap; + + refcount_set(&msg->refcnt, 2); + + req_head->msg_type = virtio_cpu_to_le(msg_type, req_head->msg_type); + + return msg; + +err_free_msg_req: + devm_kfree(dev, msg->req); + +err_free_msg: + devm_kfree(dev, msg); + + return NULL; +} + +/** + * viortc_msg_release() - Decrement message refcnt, potentially free message. + * @msg: message requested by driver + * + * Context: Any context. + */ +static void viortc_msg_release(struct viortc_msg *msg) +{ + struct device *dev; + + if (refcount_dec_and_test(&msg->refcnt)) { + dev = &msg->viortc->vdev->dev; + + devm_kfree(dev, msg->req); + devm_kfree(dev, msg->resp); + devm_kfree(dev, msg); + } +} + +/** + * viortc_do_cb() - generic virtqueue callback logic + * @vq: virtqueue + * @handle_buf: function to process a used buffer + * + * Context: virtqueue callback, typically interrupt. Takes and releases vq lock. + */ +static void viortc_do_cb(struct virtqueue *vq, + void (*handle_buf)(void *token, unsigned int len, + struct virtqueue *vq, + struct viortc_vq *viortc_vq, + struct viortc_dev *viortc)) +{ + struct viortc_dev *viortc = vq->vdev->priv; + struct viortc_vq *viortc_vq; + bool cb_enabled = true; + unsigned long flags; + unsigned int len; + void *token; + + viortc_vq = &viortc->vqs[vq->index]; + + for (;;) { + spin_lock_irqsave(&viortc_vq->lock, flags); + + if (cb_enabled) { + virtqueue_disable_cb(vq); + cb_enabled = false; + } + + token = virtqueue_get_buf(vq, &len); + if (!token) { + if (virtqueue_enable_cb(vq)) { + spin_unlock_irqrestore(&viortc_vq->lock, flags); + return; + } + cb_enabled = true; + } + + spin_unlock_irqrestore(&viortc_vq->lock, flags); + + if (token) + handle_buf(token, len, vq, viortc_vq, viortc); + } +} + +/** + * viortc_requestq_hdlr() - process a requestq used buffer + * @token: token identifying the buffer + * @len: bytes written by device + * @vq: virtqueue + * @viortc_vq: device specific data for virtqueue + * @viortc: device data + * + * Signals completion for each received message. + * + * Context: virtqueue callback + */ +static void viortc_requestq_hdlr(void *token, unsigned int len, + struct virtqueue *vq, + struct viortc_vq *viortc_vq, + struct viortc_dev *viortc) +{ + struct viortc_msg *msg = token; + + msg->resp_actual_size = len; + + complete(&msg->responded); + viortc_msg_release(msg); +} + +/** + * viortc_cb_requestq() - callback for requestq + * @vq: virtqueue + * + * Context: virtqueue callback + */ +static void viortc_cb_requestq(struct virtqueue *vq) +{ + viortc_do_cb(vq, viortc_requestq_hdlr); +} + +/** + * viortc_alarmq_hdlr() - process an alarmq used buffer + * @token: token identifying the buffer + * @len: bytes written by device + * @vq: virtqueue + * @viortc_vq: device specific data for virtqueue + * @viortc: device data + * + * Processes a VIRTIO_RTC_NOTIF_ALARM notification by calling the RTC class + * driver. Makes the buffer available again. + * + * Context: virtqueue callback + */ +static void viortc_alarmq_hdlr(void *token, unsigned int len, + struct virtqueue *vq, + struct viortc_vq *viortc_vq, + struct viortc_dev *viortc) +{ + struct virtio_rtc_notif_alarm *notif = token; + struct virtio_rtc_notif_head *head = token; + unsigned long flags; + u16 clock_id; + bool notify; + + if (len < sizeof(*head)) { + dev_err_ratelimited(&viortc->vdev->dev, + "%s: ignoring notification with short header\n", + __func__); + goto feed_vq; + } + + if (virtio_le_to_cpu(head->msg_type) != VIRTIO_RTC_NOTIF_ALARM) { + dev_err_ratelimited(&viortc->vdev->dev, + "%s: ignoring unknown notification type 0x%x\n", + __func__, virtio_le_to_cpu(head->msg_type)); + goto feed_vq; + } + + if (len < sizeof(*notif)) { + dev_err_ratelimited(&viortc->vdev->dev, + "%s: ignoring too small alarm notification\n", + __func__); + goto feed_vq; + } + + clock_id = virtio_le_to_cpu(notif->clock_id); + + if (!viortc->viortc_class) + dev_warn_ratelimited(&viortc->vdev->dev, + "ignoring alarm, no RTC class device available\n"); + else + viortc_class_alarm(viortc->viortc_class, clock_id); + +feed_vq: + spin_lock_irqsave(&viortc_vq->lock, flags); + + if (viortc_feed_vq(viortc, vq, notif, VIORTC_ALARMQ_BUF_CAP, token)) + dev_warn(&viortc->vdev->dev, + "%s: failed to re-expose input buffer\n", __func__); + + notify = virtqueue_kick_prepare(vq); + + spin_unlock_irqrestore(&viortc_vq->lock, flags); + + if (notify) + virtqueue_notify(vq); +} + +/** + * viortc_cb_alarmq() - callback for alarmq + * @vq: virtqueue + * + * Context: virtqueue callback + */ +static void viortc_cb_alarmq(struct virtqueue *vq) +{ + viortc_do_cb(vq, viortc_alarmq_hdlr); +} + +/** + * viortc_get_resp_errno() - converts virtio_rtc errnos to system errnos + * @resp_head: message response header + * + * Return: negative system errno, or 0 + */ +static int viortc_get_resp_errno(struct virtio_rtc_resp_head *resp_head) +{ + switch (virtio_le_to_cpu(resp_head->status)) { + case VIRTIO_RTC_S_OK: + return 0; + case VIRTIO_RTC_S_EOPNOTSUPP: + return -EOPNOTSUPP; + case VIRTIO_RTC_S_EINVAL: + return -EINVAL; + case VIRTIO_RTC_S_ENODEV: + return -ENODEV; + case VIRTIO_RTC_S_EIO: + default: + return -EIO; + } +} + +/** + * viortc_msg_xfer() - send message request, wait until message response + * @vq: virtqueue + * @msg: message with driver request + * @timeout_jiffies: message response timeout, 0 for no timeout + * + * Context: Process context. Takes and releases vq.lock. May sleep. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_msg_xfer(struct viortc_vq *vq, struct viortc_msg *msg, + unsigned long timeout_jiffies) +{ + struct scatterlist out_sg[1]; + struct scatterlist in_sg[1]; + struct scatterlist *sgs[2]; + unsigned long flags; + long timeout_ret; + bool notify; + int ret; + + sgs[0] = out_sg; + sgs[1] = in_sg; + + sg_init_one(out_sg, msg->req, msg->req_size); + sg_init_one(in_sg, msg->resp, msg->resp_cap); + + spin_lock_irqsave(&vq->lock, flags); + + ret = virtqueue_add_sgs(vq->vq, sgs, 1, 1, msg, GFP_ATOMIC); + if (ret) { + spin_unlock_irqrestore(&vq->lock, flags); + /* + * Release in place of the response callback, which will never + * come. + */ + viortc_msg_release(msg); + return ret; + } + + notify = virtqueue_kick_prepare(vq->vq); + + spin_unlock_irqrestore(&vq->lock, flags); + + if (notify) + virtqueue_notify(vq->vq); + + if (timeout_jiffies) { + timeout_ret = wait_for_completion_interruptible_timeout( + &msg->responded, timeout_jiffies); + + if (!timeout_ret) + return -ETIMEDOUT; + else if (timeout_ret < 0) + return (int)timeout_ret; + } else { + ret = wait_for_completion_interruptible(&msg->responded); + if (ret) + return ret; + } + + if (msg->resp_actual_size < sizeof(struct virtio_rtc_resp_head)) + return -EINVAL; + + ret = viortc_get_resp_errno(msg->resp); + if (ret) + return ret; + + /* + * There is not yet a case where returning a short message would make + * sense, so consider any deviation an error. + */ + if (msg->resp_actual_size != msg->resp_cap) + return -EINVAL; + + return 0; +} + +/* + * common message handle macros for messages of different types + */ + +/** + * VIORTC_DECLARE_MSG_HDL_ONSTACK() - declare message handle on stack + * @hdl: message handle name + * @msg_id: message type id + * @msg_req: message request type + * @msg_resp: message response type + */ +#define VIORTC_DECLARE_MSG_HDL_ONSTACK(hdl, msg_id, msg_req, msg_resp) \ + struct { \ + struct viortc_msg *msg; \ + msg_req *req; \ + msg_resp *resp; \ + unsigned int req_size; \ + unsigned int resp_cap; \ + u16 msg_type; \ + } hdl = { \ + NULL, NULL, NULL, sizeof(msg_req), sizeof(msg_resp), (msg_id), \ + } + +/** + * VIORTC_MSG() - extract message from message handle + * @hdl: message handle + * + * Return: struct viortc_msg + */ +#define VIORTC_MSG(hdl) ((hdl).msg) + +/** + * VIORTC_MSG_INIT() - initialize message handle + * @hdl: message handle + * @viortc: device data (struct viortc_dev *) + * + * Context: Process context. + * Return: 0 on success, -ENOMEM otherwise. + */ +#define VIORTC_MSG_INIT(hdl, viortc) \ + ({ \ + typeof(hdl) *_hdl = &(hdl); \ + \ + _hdl->msg = viortc_msg_init((viortc), _hdl->msg_type, \ + _hdl->req_size, _hdl->resp_cap); \ + if (_hdl->msg) { \ + _hdl->req = _hdl->msg->req; \ + _hdl->resp = _hdl->msg->resp; \ + } \ + _hdl->msg ? 0 : -ENOMEM; \ + }) + +/** + * VIORTC_MSG_WRITE() - write a request message field + * @hdl: message handle + * @dest_member: request message field name + * @src_ptr: pointer to data of compatible type + * + * Writes the field in little-endian format. + */ +#define VIORTC_MSG_WRITE(hdl, dest_member, src_ptr) \ + do { \ + typeof(hdl) _hdl = (hdl); \ + typeof(src_ptr) _src_ptr = (src_ptr); \ + \ + /* Sanity check: must match the member's type */ \ + typecheck(typeof(virtio_le_to_cpu(_hdl.req->dest_member)), \ + *_src_ptr); \ + \ + _hdl.req->dest_member = \ + virtio_cpu_to_le(*_src_ptr, _hdl.req->dest_member); \ + } while (0) + +/** + * VIORTC_MSG_READ() - read from a response message field + * @hdl: message handle + * @src_member: response message field name + * @dest_ptr: pointer to data of compatible type + * + * Converts from little-endian format and writes to dest_ptr. + */ +#define VIORTC_MSG_READ(hdl, src_member, dest_ptr) \ + do { \ + typeof(dest_ptr) _dest_ptr = (dest_ptr); \ + \ + /* Sanity check: must match the member's type */ \ + typecheck(typeof(virtio_le_to_cpu((hdl).resp->src_member)), \ + *_dest_ptr); \ + \ + *_dest_ptr = virtio_le_to_cpu((hdl).resp->src_member); \ + } while (0) + +/* + * read requests + */ + +/** timeout for clock readings, where timeouts are considered non-fatal */ +#define VIORTC_MSG_READ_TIMEOUT secs_to_jiffies(60) + +/** + * viortc_read() - VIRTIO_RTC_REQ_READ wrapper + * @viortc: device data + * @vio_clk_id: virtio_rtc clock id + * @reading: clock reading [ns] + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +int viortc_read(struct viortc_dev *viortc, u16 vio_clk_id, u64 *reading) +{ + VIORTC_DECLARE_MSG_HDL_ONSTACK(hdl, VIRTIO_RTC_REQ_READ, + struct virtio_rtc_req_read, + struct virtio_rtc_resp_read); + int ret; + + ret = VIORTC_MSG_INIT(hdl, viortc); + if (ret) + return ret; + + VIORTC_MSG_WRITE(hdl, clock_id, &vio_clk_id); + + ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), + VIORTC_MSG_READ_TIMEOUT); + if (ret) { + dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, + ret); + goto out_release; + } + + VIORTC_MSG_READ(hdl, clock_reading, reading); + +out_release: + viortc_msg_release(VIORTC_MSG(hdl)); + + return ret; +} + +/** + * viortc_read_cross() - VIRTIO_RTC_REQ_READ_CROSS wrapper + * @viortc: device data + * @vio_clk_id: virtio_rtc clock id + * @hw_counter: virtio_rtc HW counter type + * @reading: clock reading [ns] + * @cycles: HW counter cycles during clock reading + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +int viortc_read_cross(struct viortc_dev *viortc, u16 vio_clk_id, u8 hw_counter, + u64 *reading, u64 *cycles) +{ + VIORTC_DECLARE_MSG_HDL_ONSTACK(hdl, VIRTIO_RTC_REQ_READ_CROSS, + struct virtio_rtc_req_read_cross, + struct virtio_rtc_resp_read_cross); + int ret; + + ret = VIORTC_MSG_INIT(hdl, viortc); + if (ret) + return ret; + + VIORTC_MSG_WRITE(hdl, clock_id, &vio_clk_id); + VIORTC_MSG_WRITE(hdl, hw_counter, &hw_counter); + + ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), + VIORTC_MSG_READ_TIMEOUT); + if (ret) { + dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, + ret); + goto out_release; + } + + VIORTC_MSG_READ(hdl, clock_reading, reading); + VIORTC_MSG_READ(hdl, counter_cycles, cycles); + +out_release: + viortc_msg_release(VIORTC_MSG(hdl)); + + return ret; +} + +/* + * control requests + */ + +/** + * viortc_cfg() - VIRTIO_RTC_REQ_CFG wrapper + * @viortc: device data + * @num_clocks: # of virtio_rtc clocks + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_cfg(struct viortc_dev *viortc, u16 *num_clocks) +{ + VIORTC_DECLARE_MSG_HDL_ONSTACK(hdl, VIRTIO_RTC_REQ_CFG, + struct virtio_rtc_req_cfg, + struct virtio_rtc_resp_cfg); + int ret; + + ret = VIORTC_MSG_INIT(hdl, viortc); + if (ret) + return ret; + + ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), + 0); + if (ret) { + dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, + ret); + goto out_release; + } + + VIORTC_MSG_READ(hdl, num_clocks, num_clocks); + +out_release: + viortc_msg_release(VIORTC_MSG(hdl)); + + return ret; +} + +/** + * viortc_clock_cap() - VIRTIO_RTC_REQ_CLOCK_CAP wrapper + * @viortc: device data + * @vio_clk_id: virtio_rtc clock id + * @type: virtio_rtc clock type + * @leap_second_smearing: virtio_rtc smearing variant + * @flags: struct virtio_rtc_resp_clock_cap.flags + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_clock_cap(struct viortc_dev *viortc, u16 vio_clk_id, u8 *type, + u8 *leap_second_smearing, u8 *flags) +{ + VIORTC_DECLARE_MSG_HDL_ONSTACK(hdl, VIRTIO_RTC_REQ_CLOCK_CAP, + struct virtio_rtc_req_clock_cap, + struct virtio_rtc_resp_clock_cap); + int ret; + + ret = VIORTC_MSG_INIT(hdl, viortc); + if (ret) + return ret; + + VIORTC_MSG_WRITE(hdl, clock_id, &vio_clk_id); + + ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), + 0); + if (ret) { + dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, + ret); + goto out_release; + } + + VIORTC_MSG_READ(hdl, type, type); + VIORTC_MSG_READ(hdl, leap_second_smearing, leap_second_smearing); + VIORTC_MSG_READ(hdl, flags, flags); + +out_release: + viortc_msg_release(VIORTC_MSG(hdl)); + + return ret; +} + +/** + * viortc_cross_cap() - VIRTIO_RTC_REQ_CROSS_CAP wrapper + * @viortc: device data + * @vio_clk_id: virtio_rtc clock id + * @hw_counter: virtio_rtc HW counter type + * @supported: xtstamping is supported for the vio_clk_id/hw_counter pair + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +int viortc_cross_cap(struct viortc_dev *viortc, u16 vio_clk_id, u8 hw_counter, + bool *supported) +{ + VIORTC_DECLARE_MSG_HDL_ONSTACK(hdl, VIRTIO_RTC_REQ_CROSS_CAP, + struct virtio_rtc_req_cross_cap, + struct virtio_rtc_resp_cross_cap); + u8 flags; + int ret; + + ret = VIORTC_MSG_INIT(hdl, viortc); + if (ret) + return ret; + + VIORTC_MSG_WRITE(hdl, clock_id, &vio_clk_id); + VIORTC_MSG_WRITE(hdl, hw_counter, &hw_counter); + + ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), + 0); + if (ret) { + dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, + ret); + goto out_release; + } + + VIORTC_MSG_READ(hdl, flags, &flags); + *supported = !!(flags & VIRTIO_RTC_FLAG_CROSS_CAP); + +out_release: + viortc_msg_release(VIORTC_MSG(hdl)); + + return ret; +} + +/** + * viortc_read_alarm() - VIRTIO_RTC_REQ_READ_ALARM wrapper + * @viortc: device data + * @vio_clk_id: virtio_rtc clock id + * @alarm_time: alarm time in ns + * @enabled: whether alarm is enabled + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +int viortc_read_alarm(struct viortc_dev *viortc, u16 vio_clk_id, + u64 *alarm_time, bool *enabled) +{ + VIORTC_DECLARE_MSG_HDL_ONSTACK(hdl, VIRTIO_RTC_REQ_READ_ALARM, + struct virtio_rtc_req_read_alarm, + struct virtio_rtc_resp_read_alarm); + u8 flags; + int ret; + + ret = VIORTC_MSG_INIT(hdl, viortc); + if (ret) + return ret; + + VIORTC_MSG_WRITE(hdl, clock_id, &vio_clk_id); + + ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), + 0); + if (ret) { + dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, + ret); + goto out_release; + } + + VIORTC_MSG_READ(hdl, alarm_time, alarm_time); + VIORTC_MSG_READ(hdl, flags, &flags); + + *enabled = !!(flags & VIRTIO_RTC_FLAG_ALARM_ENABLED); + +out_release: + viortc_msg_release(VIORTC_MSG(hdl)); + + return ret; +} + +/** + * viortc_set_alarm() - VIRTIO_RTC_REQ_SET_ALARM wrapper + * @viortc: device data + * @vio_clk_id: virtio_rtc clock id + * @alarm_time: alarm time in ns + * @alarm_enable: enable or disable alarm + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +int viortc_set_alarm(struct viortc_dev *viortc, u16 vio_clk_id, u64 alarm_time, + bool alarm_enable) +{ + VIORTC_DECLARE_MSG_HDL_ONSTACK(hdl, VIRTIO_RTC_REQ_SET_ALARM, + struct virtio_rtc_req_set_alarm, + struct virtio_rtc_resp_set_alarm); + u8 flags = 0; + int ret; + + ret = VIORTC_MSG_INIT(hdl, viortc); + if (ret) + return ret; + + if (alarm_enable) + flags |= VIRTIO_RTC_FLAG_ALARM_ENABLED; + + VIORTC_MSG_WRITE(hdl, clock_id, &vio_clk_id); + VIORTC_MSG_WRITE(hdl, alarm_time, &alarm_time); + VIORTC_MSG_WRITE(hdl, flags, &flags); + + ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), + 0); + if (ret) { + dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, + ret); + goto out_release; + } + +out_release: + viortc_msg_release(VIORTC_MSG(hdl)); + + return ret; +} + +/** + * viortc_set_alarm_enabled() - VIRTIO_RTC_REQ_SET_ALARM_ENABLED wrapper + * @viortc: device data + * @vio_clk_id: virtio_rtc clock id + * @alarm_enable: enable or disable alarm + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +int viortc_set_alarm_enabled(struct viortc_dev *viortc, u16 vio_clk_id, + bool alarm_enable) +{ + VIORTC_DECLARE_MSG_HDL_ONSTACK(hdl, VIRTIO_RTC_REQ_SET_ALARM_ENABLED, + struct virtio_rtc_req_set_alarm_enabled, + struct virtio_rtc_resp_set_alarm_enabled); + u8 flags = 0; + int ret; + + ret = VIORTC_MSG_INIT(hdl, viortc); + if (ret) + return ret; + + if (alarm_enable) + flags |= VIRTIO_RTC_FLAG_ALARM_ENABLED; + + VIORTC_MSG_WRITE(hdl, clock_id, &vio_clk_id); + VIORTC_MSG_WRITE(hdl, flags, &flags); + + ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), + 0); + if (ret) { + dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, + ret); + goto out_release; + } + +out_release: + viortc_msg_release(VIORTC_MSG(hdl)); + + return ret; +} + +/* + * init, deinit + */ + +/** + * viortc_init_rtc_class_clock() - init and register a RTC class device + * @viortc: device data + * @vio_clk_id: virtio_rtc clock id + * @clock_type: virtio_rtc clock type + * @flags: struct virtio_rtc_resp_clock_cap.flags + * + * The clock must be a UTC-like clock. + * + * Context: Process context. + * Return: Positive if registered, zero if not supported by configuration, + * negative error code otherwise. + */ +static int viortc_init_rtc_class_clock(struct viortc_dev *viortc, + u16 vio_clk_id, u8 clock_type, u8 flags) +{ + struct virtio_device *vdev = viortc->vdev; + struct viortc_class *viortc_class; + struct device *dev = &vdev->dev; + bool have_alarm; + + if (clock_type != VIRTIO_RTC_CLOCK_UTC_SMEARED) { + dev_info(dev, + "not creating RTC class device for clock %d, which may step on leap seconds\n", + vio_clk_id); + return 0; + } + + if (viortc->viortc_class) { + dev_warn_once(dev, + "multiple UTC-like clocks are present, but creating only one RTC class device\n"); + return 0; + } + + have_alarm = viortc_alarms_supported(vdev) && + !!(flags & VIRTIO_RTC_FLAG_ALARM_CAP); + + viortc_class = viortc_class_init(viortc, vio_clk_id, have_alarm, dev); + if (IS_ERR(viortc_class)) + return PTR_ERR(viortc_class); + + viortc->viortc_class = viortc_class; + + if (have_alarm) + devm_device_init_wakeup(dev); + + return viortc_class_register(viortc_class) ?: 1; +} + +/** + * viortc_init_ptp_clock() - init and register PTP clock + * @viortc: device data + * @vio_clk_id: virtio_rtc clock id + * @clock_type: virtio_rtc clock type + * @leap_second_smearing: virtio_rtc leap second smearing + * + * Context: Process context. + * Return: Positive if registered, zero if not supported by configuration, + * negative error code otherwise. + */ +static int viortc_init_ptp_clock(struct viortc_dev *viortc, u16 vio_clk_id, + u8 clock_type, u8 leap_second_smearing) +{ + struct device *dev = &viortc->vdev->dev; + char ptp_clock_name[PTP_CLOCK_NAME_LEN]; + struct viortc_ptp_clock *vio_ptp; + + snprintf(ptp_clock_name, PTP_CLOCK_NAME_LEN, + "Virtio PTP type %hhu/variant %hhu", clock_type, + leap_second_smearing); + + vio_ptp = viortc_ptp_register(viortc, dev, vio_clk_id, ptp_clock_name); + if (IS_ERR(vio_ptp)) { + dev_err(dev, "failed to register PTP clock '%s'\n", + ptp_clock_name); + return PTR_ERR(vio_ptp); + } + + viortc->clocks_to_unregister[vio_clk_id] = vio_ptp; + + return !!vio_ptp; +} + +/** + * viortc_init_clock() - init local representation of virtio_rtc clock + * @viortc: device data + * @vio_clk_id: virtio_rtc clock id + * + * Initializes PHC and/or RTC class device to represent virtio_rtc clock. + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_init_clock(struct viortc_dev *viortc, u16 vio_clk_id) +{ + u8 clock_type, leap_second_smearing, flags; + bool is_exposed = false; + int ret; + + ret = viortc_clock_cap(viortc, vio_clk_id, &clock_type, + &leap_second_smearing, &flags); + if (ret) + return ret; + + if (IS_ENABLED(CONFIG_VIRTIO_RTC_CLASS) && + (clock_type == VIRTIO_RTC_CLOCK_UTC || + clock_type == VIRTIO_RTC_CLOCK_UTC_SMEARED || + clock_type == VIRTIO_RTC_CLOCK_UTC_MAYBE_SMEARED)) { + ret = viortc_init_rtc_class_clock(viortc, vio_clk_id, + clock_type, flags); + if (ret < 0) + return ret; + if (ret > 0) + is_exposed = true; + } + + if (IS_ENABLED(CONFIG_VIRTIO_RTC_PTP)) { + ret = viortc_init_ptp_clock(viortc, vio_clk_id, clock_type, + leap_second_smearing); + if (ret < 0) + return ret; + if (ret > 0) + is_exposed = true; + } + + if (!is_exposed) + dev_warn(&viortc->vdev->dev, + "cannot expose clock %d (type %d, variant %d) to userspace\n", + vio_clk_id, clock_type, leap_second_smearing); + + return 0; +} + +/** + * viortc_clocks_deinit() - unregister PHCs, stop RTC ops + * @viortc: device data + */ +static void viortc_clocks_deinit(struct viortc_dev *viortc) +{ + struct viortc_ptp_clock *vio_ptp; + unsigned int i; + + for (i = 0; i < viortc->num_clocks; i++) { + vio_ptp = viortc->clocks_to_unregister[i]; + + if (!vio_ptp) + continue; + + viortc->clocks_to_unregister[i] = NULL; + + WARN_ON(viortc_ptp_unregister(vio_ptp, &viortc->vdev->dev)); + } + + if (viortc->viortc_class) + viortc_class_stop(viortc->viortc_class); +} + +/** + * viortc_clocks_init() - init local representations of virtio_rtc clocks + * @viortc: device data + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_clocks_init(struct viortc_dev *viortc) +{ + u16 num_clocks; + unsigned int i; + int ret; + + ret = viortc_cfg(viortc, &num_clocks); + if (ret) + return ret; + + if (num_clocks < 1) { + dev_err(&viortc->vdev->dev, "device reported 0 clocks\n"); + return -ENODEV; + } + + viortc->num_clocks = num_clocks; + + viortc->clocks_to_unregister = + devm_kcalloc(&viortc->vdev->dev, num_clocks, + sizeof(*viortc->clocks_to_unregister), GFP_KERNEL); + if (!viortc->clocks_to_unregister) + return -ENOMEM; + + for (i = 0; i < num_clocks; i++) { + ret = viortc_init_clock(viortc, i); + if (ret) + goto err_deinit_clocks; + } + + return 0; + +err_deinit_clocks: + viortc_clocks_deinit(viortc); + + return ret; +} + +/** + * viortc_populate_vq() - populate alarmq with device-writable buffers + * @viortc: device data + * @viortc_vq: device specific data for virtqueue + * @buf_cap: device-writable buffer size in bytes + * @lock: lock queue during accesses + * + * Populates the alarmq with pre-allocated buffers. + * + * The caller is responsible for kicking the device. + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_populate_vq(struct viortc_dev *viortc, + struct viortc_vq *viortc_vq, u32 buf_cap, + bool lock) +{ + unsigned int num_elems, i; + struct virtqueue *vq; + unsigned long flags; + void *buf; + int ret; + + num_elems = viortc->num_alarmq_bufs; + vq = viortc_vq->vq; + + for (i = 0; i < num_elems; i++) { + buf = viortc->alarmq_bufs[i]; + + if (lock) { + spin_lock_irqsave(&viortc_vq->lock, flags); + + ret = viortc_feed_vq(viortc, vq, buf, buf_cap, buf); + + spin_unlock_irqrestore(&viortc_vq->lock, flags); + } else { + ret = viortc_feed_vq(viortc, vq, buf, buf_cap, buf); + } + + if (ret) + return ret; + } + + return 0; +} + +/** + * viortc_alloc_vq_bufs() - allocate alarmq buffers + * @viortc: device data + * @num_elems: # of buffers + * @buf_cap: per-buffer device-writable bytes + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_alloc_vq_bufs(struct viortc_dev *viortc, + unsigned int num_elems, u32 buf_cap) +{ + struct device *dev = &viortc->vdev->dev; + void **buf_list; + unsigned int i; + void *buf; + + buf_list = devm_kcalloc(dev, num_elems, sizeof(*buf_list), GFP_KERNEL); + if (!buf_list) + return -ENOMEM; + + viortc->alarmq_bufs = buf_list; + viortc->num_alarmq_bufs = num_elems; + + for (i = 0; i < num_elems; i++) { + buf = devm_kzalloc(dev, buf_cap, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + buf_list[i] = buf; + } + + return 0; +} + +/** + * viortc_init_vqs() - init virtqueues + * @viortc: device data + * + * Inits virtqueues and associated data. + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_init_vqs(struct viortc_dev *viortc) +{ + struct virtqueue *vqs[VIORTC_MAX_NR_QUEUES]; + struct virtqueue_info vqs_info[] = { + { "requestq", viortc_cb_requestq }, + { "alarmq", viortc_cb_alarmq }, + }; + struct virtio_device *vdev = viortc->vdev; + unsigned int num_elems; + int nr_queues, ret; + bool have_alarms; + + have_alarms = viortc_alarms_supported(vdev); + + if (have_alarms) + nr_queues = VIORTC_ALARMQ + 1; + else + nr_queues = VIORTC_REQUESTQ + 1; + + ret = virtio_find_vqs(vdev, nr_queues, vqs, vqs_info, NULL); + if (ret) + return ret; + + viortc->vqs[VIORTC_REQUESTQ].vq = vqs[VIORTC_REQUESTQ]; + spin_lock_init(&viortc->vqs[VIORTC_REQUESTQ].lock); + + if (have_alarms) { + viortc->vqs[VIORTC_ALARMQ].vq = vqs[VIORTC_ALARMQ]; + spin_lock_init(&viortc->vqs[VIORTC_ALARMQ].lock); + + num_elems = virtqueue_get_vring_size(vqs[VIORTC_ALARMQ]); + if (num_elems == 0) + return -ENOSPC; + + if (!viortc->alarmq_bufs) { + ret = viortc_alloc_vq_bufs(viortc, num_elems, + VIORTC_ALARMQ_BUF_CAP); + if (ret) + return ret; + } else { + viortc->num_alarmq_bufs = + min(num_elems, viortc->num_alarmq_bufs); + } + } + + return 0; +} + +/** + * viortc_probe() - probe a virtio_rtc virtio device + * @vdev: virtio device + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_probe(struct virtio_device *vdev) +{ + struct viortc_vq *alarm_viortc_vq; + struct virtqueue *alarm_vq; + struct viortc_dev *viortc; + unsigned long flags; + bool notify; + int ret; + + viortc = devm_kzalloc(&vdev->dev, sizeof(*viortc), GFP_KERNEL); + if (!viortc) + return -ENOMEM; + + vdev->priv = viortc; + viortc->vdev = vdev; + + ret = viortc_init_vqs(viortc); + if (ret) + return ret; + + virtio_device_ready(vdev); + + ret = viortc_clocks_init(viortc); + if (ret) + goto err_reset_vdev; + + if (viortc_alarms_supported(vdev)) { + alarm_viortc_vq = &viortc->vqs[VIORTC_ALARMQ]; + alarm_vq = alarm_viortc_vq->vq; + + ret = viortc_populate_vq(viortc, alarm_viortc_vq, + VIORTC_ALARMQ_BUF_CAP, true); + if (ret) + goto err_deinit_clocks; + + spin_lock_irqsave(&alarm_viortc_vq->lock, flags); + notify = virtqueue_kick_prepare(alarm_vq); + spin_unlock_irqrestore(&alarm_viortc_vq->lock, flags); + + if (notify && !virtqueue_notify(alarm_vq)) { + ret = -EIO; + goto err_deinit_clocks; + } + } + + return 0; + +err_deinit_clocks: + viortc_clocks_deinit(viortc); + +err_reset_vdev: + virtio_reset_device(vdev); + vdev->config->del_vqs(vdev); + + return ret; +} + +/** + * viortc_remove() - remove a virtio_rtc virtio device + * @vdev: virtio device + */ +static void viortc_remove(struct virtio_device *vdev) +{ + struct viortc_dev *viortc = vdev->priv; + + viortc_clocks_deinit(viortc); + + virtio_reset_device(vdev); + vdev->config->del_vqs(vdev); +} + +static int viortc_freeze(struct virtio_device *dev) +{ + /* + * Do not reset the device, so that the device may still wake up the + * system through an alarmq notification. + */ + + return 0; +} + +static int viortc_restore(struct virtio_device *dev) +{ + struct viortc_dev *viortc = dev->priv; + struct viortc_vq *alarm_viortc_vq; + struct virtqueue *alarm_vq; + bool notify = false; + int ret; + + ret = viortc_init_vqs(viortc); + if (ret) + return ret; + + alarm_viortc_vq = &viortc->vqs[VIORTC_ALARMQ]; + alarm_vq = alarm_viortc_vq->vq; + + if (viortc_alarms_supported(dev)) { + ret = viortc_populate_vq(viortc, alarm_viortc_vq, + VIORTC_ALARMQ_BUF_CAP, false); + if (ret) + return ret; + + notify = virtqueue_kick_prepare(alarm_vq); + } + + virtio_device_ready(dev); + + if (notify && !virtqueue_notify(alarm_vq)) + ret = -EIO; + + return ret; +} + +static unsigned int features[] = { +#if IS_ENABLED(CONFIG_VIRTIO_RTC_CLASS) + VIRTIO_RTC_F_ALARM, +#endif +}; + +static struct virtio_device_id id_table[] = { + { VIRTIO_ID_CLOCK, VIRTIO_DEV_ANY_ID }, + { 0 }, +}; +MODULE_DEVICE_TABLE(virtio, id_table); + +static struct virtio_driver virtio_rtc_drv = { + .driver.name = KBUILD_MODNAME, + .feature_table = features, + .feature_table_size = ARRAY_SIZE(features), + .id_table = id_table, + .probe = viortc_probe, + .remove = viortc_remove, + .freeze = pm_sleep_ptr(viortc_freeze), + .restore = pm_sleep_ptr(viortc_restore), +}; + +module_virtio_driver(virtio_rtc_drv); + +MODULE_DESCRIPTION("Virtio RTC driver"); +MODULE_AUTHOR("Qualcomm Innovation Center, Inc."); +MODULE_LICENSE("GPL"); diff --git a/drivers/virtio/virtio_rtc_internal.h b/drivers/virtio/virtio_rtc_internal.h new file mode 100644 index 000000000000..296afee6719b --- /dev/null +++ b/drivers/virtio/virtio_rtc_internal.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * virtio_rtc internal interfaces + * + * Copyright (C) 2022-2023 OpenSynergy GmbH + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _VIRTIO_RTC_INTERNAL_H_ +#define _VIRTIO_RTC_INTERNAL_H_ + +#include <linux/device.h> +#include <linux/err.h> +#include <linux/ptp_clock_kernel.h> +#include <linux/types.h> + +/* driver core IFs */ + +struct viortc_dev; + +int viortc_read(struct viortc_dev *viortc, u16 vio_clk_id, u64 *reading); +int viortc_read_cross(struct viortc_dev *viortc, u16 vio_clk_id, u8 hw_counter, + u64 *reading, u64 *cycles); +int viortc_cross_cap(struct viortc_dev *viortc, u16 vio_clk_id, u8 hw_counter, + bool *supported); +int viortc_read_alarm(struct viortc_dev *viortc, u16 vio_clk_id, + u64 *alarm_time, bool *enabled); +int viortc_set_alarm(struct viortc_dev *viortc, u16 vio_clk_id, u64 alarm_time, + bool alarm_enable); +int viortc_set_alarm_enabled(struct viortc_dev *viortc, u16 vio_clk_id, + bool alarm_enable); + +struct viortc_class; + +struct viortc_class *viortc_class_from_dev(struct device *dev); + +/* PTP IFs */ + +struct viortc_ptp_clock; + +#if IS_ENABLED(CONFIG_VIRTIO_RTC_PTP) + +struct viortc_ptp_clock *viortc_ptp_register(struct viortc_dev *viortc, + struct device *parent_dev, + u16 vio_clk_id, + const char *ptp_clock_name); +int viortc_ptp_unregister(struct viortc_ptp_clock *vio_ptp, + struct device *parent_dev); + +#else + +static inline struct viortc_ptp_clock * +viortc_ptp_register(struct viortc_dev *viortc, struct device *parent_dev, + u16 vio_clk_id, const char *ptp_clock_name) +{ + return NULL; +} + +static inline int viortc_ptp_unregister(struct viortc_ptp_clock *vio_ptp, + struct device *parent_dev) +{ + return -ENODEV; +} + +#endif + +/* HW counter IFs */ + +/** + * viortc_hw_xtstamp_params() - get HW-specific xtstamp params + * @hw_counter: virtio_rtc HW counter type + * @cs_id: clocksource id corresponding to hw_counter + * + * Gets the HW-specific xtstamp params. Returns an error if the driver cannot + * support xtstamp. + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +int viortc_hw_xtstamp_params(u8 *hw_counter, enum clocksource_ids *cs_id); + +/* RTC class IFs */ + +#if IS_ENABLED(CONFIG_VIRTIO_RTC_CLASS) + +void viortc_class_alarm(struct viortc_class *viortc_class, u16 vio_clk_id); + +void viortc_class_stop(struct viortc_class *viortc_class); + +int viortc_class_register(struct viortc_class *viortc_class); + +struct viortc_class *viortc_class_init(struct viortc_dev *viortc, + u16 vio_clk_id, bool have_alarm, + struct device *parent_dev); + +#else /* CONFIG_VIRTIO_RTC_CLASS */ + +static inline void viortc_class_alarm(struct viortc_class *viortc_class, + u16 vio_clk_id) +{ +} + +static inline void viortc_class_stop(struct viortc_class *viortc_class) +{ +} + +static inline int viortc_class_register(struct viortc_class *viortc_class) +{ + return -ENODEV; +} + +static inline struct viortc_class *viortc_class_init(struct viortc_dev *viortc, + u16 vio_clk_id, + bool have_alarm, + struct device *parent_dev) +{ + return ERR_PTR(-ENODEV); +} + +#endif /* CONFIG_VIRTIO_RTC_CLASS */ + +#endif /* _VIRTIO_RTC_INTERNAL_H_ */ diff --git a/drivers/virtio/virtio_rtc_ptp.c b/drivers/virtio/virtio_rtc_ptp.c new file mode 100644 index 000000000000..f84599950cd4 --- /dev/null +++ b/drivers/virtio/virtio_rtc_ptp.c @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Expose virtio_rtc clocks as PTP clocks. + * + * Copyright (C) 2022-2023 OpenSynergy GmbH + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. + * + * Derived from ptp_kvm_common.c, virtual PTP 1588 clock for use with KVM + * guests. + * + * Copyright (C) 2017 Red Hat Inc. + */ + +#include <linux/device.h> +#include <linux/err.h> +#include <linux/ptp_clock_kernel.h> + +#include <uapi/linux/virtio_rtc.h> + +#include "virtio_rtc_internal.h" + +/** + * struct viortc_ptp_clock - PTP clock abstraction + * @ptp_clock: PTP clock handle for unregistering + * @viortc: virtio_rtc device data + * @ptp_info: PTP clock description + * @vio_clk_id: virtio_rtc clock id + * @have_cross: device supports crosststamp with available HW counter + */ +struct viortc_ptp_clock { + struct ptp_clock *ptp_clock; + struct viortc_dev *viortc; + struct ptp_clock_info ptp_info; + u16 vio_clk_id; + bool have_cross; +}; + +/** + * struct viortc_ptp_cross_ctx - context for get_device_system_crosststamp() + * @device_time: device clock reading + * @system_counterval: HW counter value at device_time + * + * Provides the already obtained crosststamp to get_device_system_crosststamp(). + */ +struct viortc_ptp_cross_ctx { + ktime_t device_time; + struct system_counterval_t system_counterval; +}; + +/* Weak function in case get_device_system_crosststamp() is not supported */ +int __weak viortc_hw_xtstamp_params(u8 *hw_counter, enum clocksource_ids *cs_id) +{ + return -EOPNOTSUPP; +} + +/** + * viortc_ptp_get_time_fn() - callback for get_device_system_crosststamp() + * @device_time: device clock reading + * @system_counterval: HW counter value at device_time + * @ctx: context with already obtained crosststamp + * + * Return: zero (success). + */ +static int viortc_ptp_get_time_fn(ktime_t *device_time, + struct system_counterval_t *system_counterval, + void *ctx) +{ + struct viortc_ptp_cross_ctx *vio_ctx = ctx; + + *device_time = vio_ctx->device_time; + *system_counterval = vio_ctx->system_counterval; + + return 0; +} + +/** + * viortc_ptp_do_xtstamp() - get crosststamp from device + * @vio_ptp: virtio_rtc PTP clock + * @hw_counter: virtio_rtc HW counter type + * @cs_id: clocksource id corresponding to hw_counter + * @ctx: context for get_device_system_crosststamp() + * + * Reads HW-specific crosststamp from device. + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_ptp_do_xtstamp(struct viortc_ptp_clock *vio_ptp, + u8 hw_counter, enum clocksource_ids cs_id, + struct viortc_ptp_cross_ctx *ctx) +{ + u64 max_ns, ns; + int ret; + + ctx->system_counterval.cs_id = cs_id; + + ret = viortc_read_cross(vio_ptp->viortc, vio_ptp->vio_clk_id, + hw_counter, &ns, + &ctx->system_counterval.cycles); + if (ret) + return ret; + + max_ns = (u64)ktime_to_ns(KTIME_MAX); + if (ns > max_ns) + return -EINVAL; + + ctx->device_time = ns_to_ktime(ns); + + return 0; +} + +/* + * PTP clock operations + */ + +/** + * viortc_ptp_getcrosststamp() - PTP clock getcrosststamp op + * @ptp: PTP clock info + * @xtstamp: crosststamp + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_ptp_getcrosststamp(struct ptp_clock_info *ptp, + struct system_device_crosststamp *xtstamp) +{ + struct viortc_ptp_clock *vio_ptp = + container_of(ptp, struct viortc_ptp_clock, ptp_info); + struct system_time_snapshot history_begin; + struct viortc_ptp_cross_ctx ctx; + enum clocksource_ids cs_id; + u8 hw_counter; + int ret; + + if (!vio_ptp->have_cross) + return -EOPNOTSUPP; + + ret = viortc_hw_xtstamp_params(&hw_counter, &cs_id); + if (ret) + return ret; + + ktime_get_snapshot(&history_begin); + if (history_begin.cs_id != cs_id) + return -EOPNOTSUPP; + + /* + * Getting the timestamp can take many milliseconds with a slow Virtio + * device. This is too long for viortc_ptp_get_time_fn() passed to + * get_device_system_crosststamp(), which has to usually return before + * the timekeeper seqcount increases (every tick or so). + * + * So, get the actual cross-timestamp first. + */ + ret = viortc_ptp_do_xtstamp(vio_ptp, hw_counter, cs_id, &ctx); + if (ret) + return ret; + + ret = get_device_system_crosststamp(viortc_ptp_get_time_fn, &ctx, + &history_begin, xtstamp); + if (ret) + pr_debug("%s: get_device_system_crosststamp() returned %d\n", + __func__, ret); + + return ret; +} + +/* viortc_ptp_adjfine() - unsupported PTP clock adjfine op */ +static int viortc_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) +{ + return -EOPNOTSUPP; +} + +/* viortc_ptp_adjtime() - unsupported PTP clock adjtime op */ +static int viortc_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) +{ + return -EOPNOTSUPP; +} + +/* viortc_ptp_settime64() - unsupported PTP clock settime64 op */ +static int viortc_ptp_settime64(struct ptp_clock_info *ptp, + const struct timespec64 *ts) +{ + return -EOPNOTSUPP; +} + +/* + * viortc_ptp_gettimex64() - PTP clock gettimex64 op + * + * Context: Process context. + */ +static int viortc_ptp_gettimex64(struct ptp_clock_info *ptp, + struct timespec64 *ts, + struct ptp_system_timestamp *sts) +{ + struct viortc_ptp_clock *vio_ptp = + container_of(ptp, struct viortc_ptp_clock, ptp_info); + int ret; + u64 ns; + + ptp_read_system_prets(sts); + ret = viortc_read(vio_ptp->viortc, vio_ptp->vio_clk_id, &ns); + ptp_read_system_postts(sts); + + if (ret) + return ret; + + if (ns > (u64)S64_MAX) + return -EINVAL; + + *ts = ns_to_timespec64((s64)ns); + + return 0; +} + +/* viortc_ptp_enable() - unsupported PTP clock enable op */ +static int viortc_ptp_enable(struct ptp_clock_info *ptp, + struct ptp_clock_request *rq, int on) +{ + return -EOPNOTSUPP; +} + +/* + * viortc_ptp_info_template - ptp_clock_info template + * + * The .name member will be set for individual virtio_rtc PTP clocks. + * + * The .getcrosststamp member will be cleared for PTP clocks not supporting + * crosststamp. + */ +static const struct ptp_clock_info viortc_ptp_info_template = { + .owner = THIS_MODULE, + /* .name is set according to clock type */ + .adjfine = viortc_ptp_adjfine, + .adjtime = viortc_ptp_adjtime, + .gettimex64 = viortc_ptp_gettimex64, + .settime64 = viortc_ptp_settime64, + .enable = viortc_ptp_enable, + .getcrosststamp = viortc_ptp_getcrosststamp, +}; + +/** + * viortc_ptp_unregister() - PTP clock unregistering wrapper + * @vio_ptp: virtio_rtc PTP clock + * @parent_dev: parent device of PTP clock + * + * Return: Zero on success, negative error code otherwise. + */ +int viortc_ptp_unregister(struct viortc_ptp_clock *vio_ptp, + struct device *parent_dev) +{ + int ret = ptp_clock_unregister(vio_ptp->ptp_clock); + + if (!ret) + devm_kfree(parent_dev, vio_ptp); + + return ret; +} + +/** + * viortc_ptp_get_cross_cap() - get xtstamp support info from device + * @viortc: virtio_rtc device data + * @vio_ptp: virtio_rtc PTP clock abstraction + * + * Context: Process context. + * Return: Zero on success, negative error code otherwise. + */ +static int viortc_ptp_get_cross_cap(struct viortc_dev *viortc, + struct viortc_ptp_clock *vio_ptp) +{ + enum clocksource_ids cs_id; + bool xtstamp_supported; + u8 hw_counter; + int ret; + + ret = viortc_hw_xtstamp_params(&hw_counter, &cs_id); + if (ret) { + vio_ptp->have_cross = false; + return 0; + } + + ret = viortc_cross_cap(viortc, vio_ptp->vio_clk_id, hw_counter, + &xtstamp_supported); + if (ret) + return ret; + + vio_ptp->have_cross = xtstamp_supported; + + return 0; +} + +/** + * viortc_ptp_register() - prepare and register PTP clock + * @viortc: virtio_rtc device data + * @parent_dev: parent device for PTP clock + * @vio_clk_id: id of virtio_rtc clock which backs PTP clock + * @ptp_clock_name: PTP clock name + * + * Context: Process context. + * Return: Pointer on success, ERR_PTR() otherwise; NULL if PTP clock support + * not available. + */ +struct viortc_ptp_clock *viortc_ptp_register(struct viortc_dev *viortc, + struct device *parent_dev, + u16 vio_clk_id, + const char *ptp_clock_name) +{ + struct viortc_ptp_clock *vio_ptp; + struct ptp_clock *ptp_clock; + ssize_t len; + int ret; + + vio_ptp = devm_kzalloc(parent_dev, sizeof(*vio_ptp), GFP_KERNEL); + if (!vio_ptp) + return ERR_PTR(-ENOMEM); + + vio_ptp->viortc = viortc; + vio_ptp->vio_clk_id = vio_clk_id; + vio_ptp->ptp_info = viortc_ptp_info_template; + len = strscpy(vio_ptp->ptp_info.name, ptp_clock_name, + sizeof(vio_ptp->ptp_info.name)); + if (len < 0) { + ret = len; + goto err_free_dev; + } + + ret = viortc_ptp_get_cross_cap(viortc, vio_ptp); + if (ret) + goto err_free_dev; + + if (!vio_ptp->have_cross) + vio_ptp->ptp_info.getcrosststamp = NULL; + + ptp_clock = ptp_clock_register(&vio_ptp->ptp_info, parent_dev); + if (IS_ERR(ptp_clock)) + goto err_on_register; + + vio_ptp->ptp_clock = ptp_clock; + + return vio_ptp; + +err_on_register: + ret = PTR_ERR(ptp_clock); + +err_free_dev: + devm_kfree(parent_dev, vio_ptp); + return ERR_PTR(ret); +} diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index 1f60c9d5cb18..657b07a60788 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -28,19 +28,6 @@ struct virtio_vdpa_device { struct virtio_device vdev; struct vdpa_device *vdpa; u64 features; - - /* The lock to protect virtqueue list */ - spinlock_t lock; - /* List of virtio_vdpa_vq_info */ - struct list_head virtqueues; -}; - -struct virtio_vdpa_vq_info { - /* the actual virtqueue */ - struct virtqueue *vq; - - /* the list node for the virtqueues list */ - struct list_head node; }; static inline struct virtio_vdpa_device * @@ -135,9 +122,9 @@ static irqreturn_t virtio_vdpa_config_cb(void *private) static irqreturn_t virtio_vdpa_virtqueue_cb(void *private) { - struct virtio_vdpa_vq_info *info = private; + struct virtqueue *vq = private; - return vring_interrupt(0, info->vq); + return vring_interrupt(0, vq); } static struct virtqueue * @@ -145,18 +132,15 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, void (*callback)(struct virtqueue *vq), const char *name, bool ctx) { - struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); struct vdpa_device *vdpa = vd_get_vdpa(vdev); struct device *dma_dev; const struct vdpa_config_ops *ops = vdpa->config; - struct virtio_vdpa_vq_info *info; bool (*notify)(struct virtqueue *vq) = virtio_vdpa_notify; struct vdpa_callback cb; struct virtqueue *vq; u64 desc_addr, driver_addr, device_addr; /* Assume split virtqueue, switch to packed if necessary */ struct vdpa_vq_state state = {0}; - unsigned long flags; u32 align, max_num, min_num = 1; bool may_reduce_num = true; int err; @@ -179,10 +163,6 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, if (ops->get_vq_ready(vdpa, index)) return ERR_PTR(-ENOENT); - /* Allocate and fill out our active queue description */ - info = kmalloc(sizeof(*info), GFP_KERNEL); - if (!info) - return ERR_PTR(-ENOMEM); if (ops->get_vq_size) max_num = ops->get_vq_size(vdpa, index); else @@ -217,7 +197,7 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, /* Setup virtqueue callback */ cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL; - cb.private = info; + cb.private = vq; cb.trigger = NULL; ops->set_vq_cb(vdpa, index, &cb); ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq)); @@ -248,13 +228,6 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, ops->set_vq_ready(vdpa, index, 1); - vq->priv = info; - info->vq = vq; - - spin_lock_irqsave(&vd_dev->lock, flags); - list_add(&info->node, &vd_dev->virtqueues); - spin_unlock_irqrestore(&vd_dev->lock, flags); - return vq; err_vq: @@ -263,7 +236,6 @@ error_new_virtqueue: ops->set_vq_ready(vdpa, index, 0); /* VDPA driver should make sure vq is stopeed here */ WARN_ON(ops->get_vq_ready(vdpa, index)); - kfree(info); return ERR_PTR(err); } @@ -272,20 +244,12 @@ static void virtio_vdpa_del_vq(struct virtqueue *vq) struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev); struct vdpa_device *vdpa = vd_dev->vdpa; const struct vdpa_config_ops *ops = vdpa->config; - struct virtio_vdpa_vq_info *info = vq->priv; unsigned int index = vq->index; - unsigned long flags; - - spin_lock_irqsave(&vd_dev->lock, flags); - list_del(&info->node); - spin_unlock_irqrestore(&vd_dev->lock, flags); /* Select and deactivate the queue (best effort) */ ops->set_vq_ready(vdpa, index, 0); vring_del_virtqueue(vq); - - kfree(info); } static void virtio_vdpa_del_vqs(struct virtio_device *vdev) @@ -329,20 +293,21 @@ create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd) for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) { unsigned int this_vecs = affd->set_size[i]; + unsigned int nr_masks; int j; - struct cpumask *result = group_cpus_evenly(this_vecs); + struct cpumask *result = group_cpus_evenly(this_vecs, &nr_masks); if (!result) { kfree(masks); return NULL; } - for (j = 0; j < this_vecs; j++) + for (j = 0; j < nr_masks; j++) cpumask_copy(&masks[curvec + j], &result[j]); kfree(result); - curvec += this_vecs; - usedvecs += this_vecs; + curvec += nr_masks; + usedvecs += nr_masks; } /* Fill out vectors at the end that don't need affinity */ @@ -501,8 +466,6 @@ static int virtio_vdpa_probe(struct vdpa_device *vdpa) vd_dev->vdev.dev.release = virtio_vdpa_release_dev; vd_dev->vdev.config = &virtio_vdpa_config_ops; vd_dev->vdpa = vdpa; - INIT_LIST_HEAD(&vd_dev->virtqueues); - spin_lock_init(&vd_dev->lock); vd_dev->vdev.id.device = ops->get_device_id(vdpa); if (vd_dev->vdev.id.device == 0) |