summaryrefslogtreecommitdiff
path: root/drivers/vdpa/octeon_ep/octep_vdpa_hw.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/vdpa/octeon_ep/octep_vdpa_hw.c')
-rw-r--r--drivers/vdpa/octeon_ep/octep_vdpa_hw.c517
1 files changed, 517 insertions, 0 deletions
diff --git a/drivers/vdpa/octeon_ep/octep_vdpa_hw.c b/drivers/vdpa/octeon_ep/octep_vdpa_hw.c
new file mode 100644
index 000000000000..7fa0491bb201
--- /dev/null
+++ b/drivers/vdpa/octeon_ep/octep_vdpa_hw.c
@@ -0,0 +1,517 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (C) 2024 Marvell. */
+
+#include <linux/iopoll.h>
+
+#include "octep_vdpa.h"
+
+enum octep_mbox_ids {
+ OCTEP_MBOX_MSG_SET_VQ_STATE = 1,
+ OCTEP_MBOX_MSG_GET_VQ_STATE,
+};
+
+#define OCTEP_HW_TIMEOUT 10000000
+
+#define MBOX_OFFSET 64
+#define MBOX_RSP_MASK 0x00000001
+#define MBOX_RC_MASK 0x0000FFFE
+
+#define MBOX_RSP_TO_ERR(val) (-(((val) & MBOX_RC_MASK) >> 2))
+#define MBOX_AVAIL(val) (((val) & MBOX_RSP_MASK))
+#define MBOX_RSP(val) ((val) & (MBOX_RC_MASK | MBOX_RSP_MASK))
+
+#define DEV_RST_ACK_BIT 7
+#define FEATURE_SEL_ACK_BIT 15
+#define QUEUE_SEL_ACK_BIT 15
+
+struct octep_mbox_hdr {
+ u8 ver;
+ u8 rsvd1;
+ u16 id;
+ u16 rsvd2;
+#define MBOX_REQ_SIG (0xdead)
+#define MBOX_RSP_SIG (0xbeef)
+ u16 sig;
+};
+
+struct octep_mbox_sts {
+ u16 rsp:1;
+ u16 rc:15;
+ u16 rsvd;
+};
+
+struct octep_mbox {
+ struct octep_mbox_hdr hdr;
+ struct octep_mbox_sts sts;
+ u64 rsvd;
+ u32 data[];
+};
+
+static inline struct octep_mbox __iomem *octep_get_mbox(struct octep_hw *oct_hw)
+{
+ return (struct octep_mbox __iomem *)(oct_hw->dev_cfg + MBOX_OFFSET);
+}
+
+static inline int octep_wait_for_mbox_avail(struct octep_mbox __iomem *mbox)
+{
+ u32 val;
+
+ return readx_poll_timeout(ioread32, &mbox->sts, val, MBOX_AVAIL(val), 10,
+ OCTEP_HW_TIMEOUT);
+}
+
+static inline int octep_wait_for_mbox_rsp(struct octep_mbox __iomem *mbox)
+{
+ u32 val;
+
+ return readx_poll_timeout(ioread32, &mbox->sts, val, MBOX_RSP(val), 10,
+ OCTEP_HW_TIMEOUT);
+}
+
+static inline void octep_write_hdr(struct octep_mbox __iomem *mbox, u16 id, u16 sig)
+{
+ iowrite16(id, &mbox->hdr.id);
+ iowrite16(sig, &mbox->hdr.sig);
+}
+
+static inline u32 octep_read_sig(struct octep_mbox __iomem *mbox)
+{
+ return ioread16(&mbox->hdr.sig);
+}
+
+static inline void octep_write_sts(struct octep_mbox __iomem *mbox, u32 sts)
+{
+ iowrite32(sts, &mbox->sts);
+}
+
+static inline u32 octep_read_sts(struct octep_mbox __iomem *mbox)
+{
+ return ioread32(&mbox->sts);
+}
+
+static inline u32 octep_read32_word(struct octep_mbox __iomem *mbox, u16 word_idx)
+{
+ return ioread32(&mbox->data[word_idx]);
+}
+
+static inline void octep_write32_word(struct octep_mbox __iomem *mbox, u16 word_idx, u32 word)
+{
+ return iowrite32(word, &mbox->data[word_idx]);
+}
+
+static int octep_process_mbox(struct octep_hw *oct_hw, u16 id, u16 qid, void *buffer,
+ u32 buf_size, bool write)
+{
+ struct octep_mbox __iomem *mbox = octep_get_mbox(oct_hw);
+ struct pci_dev *pdev = oct_hw->pdev;
+ u32 *p = (u32 *)buffer;
+ u16 data_wds;
+ int ret, i;
+ u32 val;
+
+ if (!IS_ALIGNED(buf_size, 4))
+ return -EINVAL;
+
+ /* Make sure mbox space is available */
+ ret = octep_wait_for_mbox_avail(mbox);
+ if (ret) {
+ dev_warn(&pdev->dev, "Timeout waiting for previous mbox data to be consumed\n");
+ return ret;
+ }
+ data_wds = buf_size / 4;
+
+ if (write) {
+ for (i = 1; i <= data_wds; i++) {
+ octep_write32_word(mbox, i, *p);
+ p++;
+ }
+ }
+ octep_write32_word(mbox, 0, (u32)qid);
+ octep_write_sts(mbox, 0);
+
+ octep_write_hdr(mbox, id, MBOX_REQ_SIG);
+
+ ret = octep_wait_for_mbox_rsp(mbox);
+ if (ret) {
+ dev_warn(&pdev->dev, "Timeout waiting for mbox : %d response\n", id);
+ return ret;
+ }
+
+ val = octep_read_sig(mbox);
+ if ((val & 0xFFFF) != MBOX_RSP_SIG) {
+ dev_warn(&pdev->dev, "Invalid Signature from mbox : %d response\n", id);
+ return ret;
+ }
+
+ val = octep_read_sts(mbox);
+ if (val & MBOX_RC_MASK) {
+ ret = MBOX_RSP_TO_ERR(val);
+ dev_warn(&pdev->dev, "Error while processing mbox : %d, err %d\n", id, ret);
+ return ret;
+ }
+
+ if (!write)
+ for (i = 1; i <= data_wds; i++)
+ *p++ = octep_read32_word(mbox, i);
+
+ return 0;
+}
+
+static void octep_mbox_init(struct octep_mbox __iomem *mbox)
+{
+ iowrite32(1, &mbox->sts);
+}
+
+int octep_verify_features(u64 features)
+{
+ /* Minimum features to expect */
+ if (!(features & BIT_ULL(VIRTIO_F_VERSION_1)))
+ return -EOPNOTSUPP;
+
+ if (!(features & BIT_ULL(VIRTIO_F_NOTIFICATION_DATA)))
+ return -EOPNOTSUPP;
+
+ if (!(features & BIT_ULL(VIRTIO_F_RING_PACKED)))
+ return -EOPNOTSUPP;
+
+ return 0;
+}
+
+u8 octep_hw_get_status(struct octep_hw *oct_hw)
+{
+ return ioread8(&oct_hw->common_cfg->device_status);
+}
+
+void octep_hw_set_status(struct octep_hw *oct_hw, u8 status)
+{
+ iowrite8(status, &oct_hw->common_cfg->device_status);
+}
+
+void octep_hw_reset(struct octep_hw *oct_hw)
+{
+ u8 val;
+
+ octep_hw_set_status(oct_hw, 0 | BIT(DEV_RST_ACK_BIT));
+ if (readx_poll_timeout(ioread8, &oct_hw->common_cfg->device_status, val, !val, 10,
+ OCTEP_HW_TIMEOUT)) {
+ dev_warn(&oct_hw->pdev->dev, "Octeon device reset timeout\n");
+ return;
+ }
+}
+
+static int feature_sel_write_with_timeout(struct octep_hw *oct_hw, u32 select, void __iomem *addr)
+{
+ u32 val;
+
+ iowrite32(select | BIT(FEATURE_SEL_ACK_BIT), addr);
+
+ if (readx_poll_timeout(ioread32, addr, val, val == select, 10, OCTEP_HW_TIMEOUT)) {
+ dev_warn(&oct_hw->pdev->dev, "Feature select%d write timeout\n", select);
+ return -1;
+ }
+ return 0;
+}
+
+u64 octep_hw_get_dev_features(struct octep_hw *oct_hw)
+{
+ u32 features_lo, features_hi;
+
+ if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->device_feature_select))
+ return 0;
+
+ features_lo = ioread32(&oct_hw->common_cfg->device_feature);
+
+ if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->device_feature_select))
+ return 0;
+
+ features_hi = ioread32(&oct_hw->common_cfg->device_feature);
+
+ return ((u64)features_hi << 32) | features_lo;
+}
+
+u64 octep_hw_get_drv_features(struct octep_hw *oct_hw)
+{
+ u32 features_lo, features_hi;
+
+ if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->guest_feature_select))
+ return 0;
+
+ features_lo = ioread32(&oct_hw->common_cfg->guest_feature);
+
+ if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->guest_feature_select))
+ return 0;
+
+ features_hi = ioread32(&oct_hw->common_cfg->guest_feature);
+
+ return ((u64)features_hi << 32) | features_lo;
+}
+
+void octep_hw_set_drv_features(struct octep_hw *oct_hw, u64 features)
+{
+ if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->guest_feature_select))
+ return;
+
+ iowrite32(features & (BIT_ULL(32) - 1), &oct_hw->common_cfg->guest_feature);
+
+ if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->guest_feature_select))
+ return;
+
+ iowrite32(features >> 32, &oct_hw->common_cfg->guest_feature);
+}
+
+void octep_write_queue_select(struct octep_hw *oct_hw, u16 queue_id)
+{
+ u16 val;
+
+ iowrite16(queue_id | BIT(QUEUE_SEL_ACK_BIT), &oct_hw->common_cfg->queue_select);
+
+ if (readx_poll_timeout(ioread16, &oct_hw->common_cfg->queue_select, val, val == queue_id,
+ 10, OCTEP_HW_TIMEOUT)) {
+ dev_warn(&oct_hw->pdev->dev, "Queue select write timeout\n");
+ return;
+ }
+}
+
+void octep_notify_queue(struct octep_hw *oct_hw, u16 qid)
+{
+ iowrite16(qid, oct_hw->vqs[qid].notify_addr);
+}
+
+void octep_read_dev_config(struct octep_hw *oct_hw, u64 offset, void *dst, int length)
+{
+ u8 old_gen, new_gen, *p;
+ int i;
+
+ if (WARN_ON(offset + length > oct_hw->config_size))
+ return;
+
+ do {
+ old_gen = ioread8(&oct_hw->common_cfg->config_generation);
+ p = dst;
+ for (i = 0; i < length; i++)
+ *p++ = ioread8(oct_hw->dev_cfg + offset + i);
+
+ new_gen = ioread8(&oct_hw->common_cfg->config_generation);
+ } while (old_gen != new_gen);
+}
+
+int octep_set_vq_address(struct octep_hw *oct_hw, u16 qid, u64 desc_area, u64 driver_area,
+ u64 device_area)
+{
+ struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;
+
+ octep_write_queue_select(oct_hw, qid);
+ vp_iowrite64_twopart(desc_area, &cfg->queue_desc_lo,
+ &cfg->queue_desc_hi);
+ vp_iowrite64_twopart(driver_area, &cfg->queue_avail_lo,
+ &cfg->queue_avail_hi);
+ vp_iowrite64_twopart(device_area, &cfg->queue_used_lo,
+ &cfg->queue_used_hi);
+
+ return 0;
+}
+
+int octep_get_vq_state(struct octep_hw *oct_hw, u16 qid, struct vdpa_vq_state *state)
+{
+ return octep_process_mbox(oct_hw, OCTEP_MBOX_MSG_GET_VQ_STATE, qid, state,
+ sizeof(*state), 0);
+}
+
+int octep_set_vq_state(struct octep_hw *oct_hw, u16 qid, const struct vdpa_vq_state *state)
+{
+ struct vdpa_vq_state q_state;
+
+ memcpy(&q_state, state, sizeof(struct vdpa_vq_state));
+ return octep_process_mbox(oct_hw, OCTEP_MBOX_MSG_SET_VQ_STATE, qid, &q_state,
+ sizeof(*state), 1);
+}
+
+void octep_set_vq_num(struct octep_hw *oct_hw, u16 qid, u32 num)
+{
+ struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;
+
+ octep_write_queue_select(oct_hw, qid);
+ iowrite16(num, &cfg->queue_size);
+}
+
+void octep_set_vq_ready(struct octep_hw *oct_hw, u16 qid, bool ready)
+{
+ struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;
+
+ octep_write_queue_select(oct_hw, qid);
+ iowrite16(ready, &cfg->queue_enable);
+}
+
+bool octep_get_vq_ready(struct octep_hw *oct_hw, u16 qid)
+{
+ struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;
+
+ octep_write_queue_select(oct_hw, qid);
+ return ioread16(&cfg->queue_enable);
+}
+
+u16 octep_get_vq_size(struct octep_hw *oct_hw)
+{
+ octep_write_queue_select(oct_hw, 0);
+ return ioread16(&oct_hw->common_cfg->queue_size);
+}
+
+static u32 octep_get_config_size(struct octep_hw *oct_hw)
+{
+ return sizeof(struct virtio_net_config);
+}
+
+static void __iomem *octep_get_cap_addr(struct octep_hw *oct_hw, struct virtio_pci_cap *cap)
+{
+ struct device *dev = &oct_hw->pdev->dev;
+ u32 length = le32_to_cpu(cap->length);
+ u32 offset = le32_to_cpu(cap->offset);
+ u8 bar = cap->bar;
+ u32 len;
+
+ if (bar != OCTEP_HW_CAPS_BAR) {
+ dev_err(dev, "Invalid bar: %u\n", bar);
+ return NULL;
+ }
+ if (offset + length < offset) {
+ dev_err(dev, "offset(%u) + length(%u) overflows\n",
+ offset, length);
+ return NULL;
+ }
+ len = pci_resource_len(oct_hw->pdev, bar);
+ if (offset + length > len) {
+ dev_err(dev, "invalid cap: overflows bar space: %u > %u\n",
+ offset + length, len);
+ return NULL;
+ }
+ return oct_hw->base[bar] + offset;
+}
+
+/* In Octeon DPU device, the virtio config space is completely
+ * emulated by the device's firmware. So, the standard pci config
+ * read apis can't be used for reading the virtio capability.
+ */
+static void octep_pci_caps_read(struct octep_hw *oct_hw, void *buf, size_t len, off_t offset)
+{
+ u8 __iomem *bar = oct_hw->base[OCTEP_HW_CAPS_BAR];
+ u8 *p = buf;
+ size_t i;
+
+ for (i = 0; i < len; i++)
+ *p++ = ioread8(bar + offset + i);
+}
+
+static int octep_pci_signature_verify(struct octep_hw *oct_hw)
+{
+ u32 signature[2];
+
+ octep_pci_caps_read(oct_hw, &signature, sizeof(signature), 0);
+
+ if (signature[0] != OCTEP_FW_READY_SIGNATURE0)
+ return -1;
+
+ if (signature[1] != OCTEP_FW_READY_SIGNATURE1)
+ return -1;
+
+ return 0;
+}
+
+int octep_hw_caps_read(struct octep_hw *oct_hw, struct pci_dev *pdev)
+{
+ struct octep_mbox __iomem *mbox;
+ struct device *dev = &pdev->dev;
+ struct virtio_pci_cap cap;
+ u16 notify_off;
+ int i, ret;
+ u8 pos;
+
+ oct_hw->pdev = pdev;
+ ret = octep_pci_signature_verify(oct_hw);
+ if (ret) {
+ dev_err(dev, "Octeon Virtio FW is not initialized\n");
+ return -EIO;
+ }
+
+ octep_pci_caps_read(oct_hw, &pos, 1, PCI_CAPABILITY_LIST);
+
+ while (pos) {
+ octep_pci_caps_read(oct_hw, &cap, 2, pos);
+
+ if (cap.cap_vndr != PCI_CAP_ID_VNDR) {
+ dev_err(dev, "Found invalid capability vndr id: %d\n", cap.cap_vndr);
+ break;
+ }
+
+ octep_pci_caps_read(oct_hw, &cap, sizeof(cap), pos);
+
+ dev_info(dev, "[%2x] cfg type: %u, bar: %u, offset: %04x, len: %u\n",
+ pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
+
+ switch (cap.cfg_type) {
+ case VIRTIO_PCI_CAP_COMMON_CFG:
+ oct_hw->common_cfg = octep_get_cap_addr(oct_hw, &cap);
+ break;
+ case VIRTIO_PCI_CAP_NOTIFY_CFG:
+ octep_pci_caps_read(oct_hw, &oct_hw->notify_off_multiplier,
+ 4, pos + sizeof(cap));
+
+ oct_hw->notify_base = octep_get_cap_addr(oct_hw, &cap);
+ oct_hw->notify_bar = cap.bar;
+ oct_hw->notify_base_pa = pci_resource_start(pdev, cap.bar) +
+ le32_to_cpu(cap.offset);
+ break;
+ case VIRTIO_PCI_CAP_DEVICE_CFG:
+ oct_hw->dev_cfg = octep_get_cap_addr(oct_hw, &cap);
+ break;
+ case VIRTIO_PCI_CAP_ISR_CFG:
+ oct_hw->isr = octep_get_cap_addr(oct_hw, &cap);
+ break;
+ }
+
+ pos = cap.cap_next;
+ }
+ if (!oct_hw->common_cfg || !oct_hw->notify_base ||
+ !oct_hw->dev_cfg || !oct_hw->isr) {
+ dev_err(dev, "Incomplete PCI capabilities");
+ return -EIO;
+ }
+ dev_info(dev, "common cfg mapped at: 0x%016llx\n", (u64)(uintptr_t)oct_hw->common_cfg);
+ dev_info(dev, "device cfg mapped at: 0x%016llx\n", (u64)(uintptr_t)oct_hw->dev_cfg);
+ dev_info(dev, "isr cfg mapped at: 0x%016llx\n", (u64)(uintptr_t)oct_hw->isr);
+ dev_info(dev, "notify base: 0x%016llx, notify off multiplier: %u\n",
+ (u64)(uintptr_t)oct_hw->notify_base, oct_hw->notify_off_multiplier);
+
+ oct_hw->config_size = octep_get_config_size(oct_hw);
+ oct_hw->features = octep_hw_get_dev_features(oct_hw);
+
+ ret = octep_verify_features(oct_hw->features);
+ if (ret) {
+ dev_err(&pdev->dev, "Couldn't read features from the device FW\n");
+ return ret;
+ }
+ oct_hw->nr_vring = vp_ioread16(&oct_hw->common_cfg->num_queues);
+
+ oct_hw->vqs = devm_kcalloc(&pdev->dev, oct_hw->nr_vring, sizeof(*oct_hw->vqs), GFP_KERNEL);
+ if (!oct_hw->vqs)
+ return -ENOMEM;
+
+ oct_hw->irq = -1;
+
+ dev_info(&pdev->dev, "Device features : %llx\n", oct_hw->features);
+ dev_info(&pdev->dev, "Maximum queues : %u\n", oct_hw->nr_vring);
+
+ for (i = 0; i < oct_hw->nr_vring; i++) {
+ octep_write_queue_select(oct_hw, i);
+ notify_off = vp_ioread16(&oct_hw->common_cfg->queue_notify_off);
+ oct_hw->vqs[i].notify_addr = oct_hw->notify_base +
+ notify_off * oct_hw->notify_off_multiplier;
+ oct_hw->vqs[i].cb_notify_addr = (u32 __iomem *)oct_hw->vqs[i].notify_addr + 1;
+ oct_hw->vqs[i].notify_pa = oct_hw->notify_base_pa +
+ notify_off * oct_hw->notify_off_multiplier;
+ }
+ mbox = octep_get_mbox(oct_hw);
+ octep_mbox_init(mbox);
+ dev_info(dev, "mbox mapped at: 0x%016llx\n", (u64)(uintptr_t)mbox);
+
+ return 0;
+}
generated by cgit v1.2.3 (git 2.39.0) at 2025-02-10 07:24:22 +0300