summaryrefslogtreecommitdiff
path: root/drivers/vdpa
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/vdpa')
-rw-r--r--drivers/vdpa/Kconfig11
-rw-r--r--drivers/vdpa/Makefile1
-rw-r--r--drivers/vdpa/ifcvf/ifcvf_main.c1
-rw-r--r--drivers/vdpa/mlx5/net/mlx5_vnet.c429
-rw-r--r--drivers/vdpa/mlx5/net/mlx5_vnet.h1
-rw-r--r--drivers/vdpa/octeon_ep/Makefile4
-rw-r--r--drivers/vdpa/octeon_ep/octep_vdpa.h94
-rw-r--r--drivers/vdpa/octeon_ep/octep_vdpa_hw.c517
-rw-r--r--drivers/vdpa/octeon_ep/octep_vdpa_main.c857
-rw-r--r--drivers/vdpa/vdpa.c1
10 files changed, 1777 insertions, 139 deletions
diff --git a/drivers/vdpa/Kconfig b/drivers/vdpa/Kconfig
index 656c1cb541de..5265d09fc1c4 100644
--- a/drivers/vdpa/Kconfig
+++ b/drivers/vdpa/Kconfig
@@ -126,4 +126,15 @@ config PDS_VDPA
With this driver, the VirtIO dataplane can be
offloaded to an AMD/Pensando DSC device.
+config OCTEONEP_VDPA
+ tristate "vDPA driver for Octeon DPU devices"
+ depends on m
+ depends on PCI_MSI
+ help
+ This is a vDPA driver designed for Marvell's Octeon DPU devices.
+ This driver enables the offloading of the VirtIO dataplane to an
+ Octeon DPU device.
+ Please note that this driver must be built as a module and it
+ cannot be loaded until the Octeon emulation software is running.
+
endif # VDPA
diff --git a/drivers/vdpa/Makefile b/drivers/vdpa/Makefile
index 8f53c6f3cca7..5654d36707af 100644
--- a/drivers/vdpa/Makefile
+++ b/drivers/vdpa/Makefile
@@ -8,3 +8,4 @@ obj-$(CONFIG_VP_VDPA) += virtio_pci/
obj-$(CONFIG_ALIBABA_ENI_VDPA) += alibaba/
obj-$(CONFIG_SNET_VDPA) += solidrun/
obj-$(CONFIG_PDS_VDPA) += pds/
+obj-$(CONFIG_OCTEONEP_VDPA) += octeon_ep/
diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
index 80d0a0460885..ccf64d7bbfaa 100644
--- a/drivers/vdpa/ifcvf/ifcvf_main.c
+++ b/drivers/vdpa/ifcvf/ifcvf_main.c
@@ -894,4 +894,5 @@ static struct pci_driver ifcvf_driver = {
module_pci_driver(ifcvf_driver);
+MODULE_DESCRIPTION("Intel IFC VF NIC driver for virtio dataplane offloading");
MODULE_LICENSE("GPL v2");
diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c
index ecfc16151d61..fa78e8288ebb 100644
--- a/drivers/vdpa/mlx5/net/mlx5_vnet.c
+++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c
@@ -48,6 +48,18 @@ MODULE_LICENSE("Dual BSD/GPL");
#define MLX5V_UNTAGGED 0x1000
+/* Device must start with 1 queue pair, as per VIRTIO v1.2 spec, section
+ * 5.1.6.5.5 "Device operation in multiqueue mode":
+ *
+ * Multiqueue is disabled by default.
+ * The driver enables multiqueue by sending a command using class
+ * VIRTIO_NET_CTRL_MQ. The command selects the mode of multiqueue
+ * operation, as follows: ...
+ */
+#define MLX5V_DEFAULT_VQ_COUNT 2
+
+#define MLX5V_DEFAULT_VQ_SIZE 256
+
struct mlx5_vdpa_cq_buf {
struct mlx5_frag_buf_ctrl fbc;
struct mlx5_frag_buf frag_buf;
@@ -144,10 +156,11 @@ static bool is_index_valid(struct mlx5_vdpa_dev *mvdev, u16 idx)
return idx <= mvdev->max_idx;
}
-static void free_resources(struct mlx5_vdpa_net *ndev);
-static void init_mvqs(struct mlx5_vdpa_net *ndev);
-static int setup_driver(struct mlx5_vdpa_dev *mvdev);
-static void teardown_driver(struct mlx5_vdpa_net *ndev);
+static void free_fixed_resources(struct mlx5_vdpa_net *ndev);
+static void mvqs_set_defaults(struct mlx5_vdpa_net *ndev);
+static int setup_vq_resources(struct mlx5_vdpa_net *ndev, bool filled);
+static void teardown_vq_resources(struct mlx5_vdpa_net *ndev);
+static int resume_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq);
static bool mlx5_vdpa_debug;
@@ -862,13 +875,16 @@ static bool msix_mode_supported(struct mlx5_vdpa_dev *mvdev)
pci_msix_can_alloc_dyn(mvdev->mdev->pdev);
}
-static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
+static int create_virtqueue(struct mlx5_vdpa_net *ndev,
+ struct mlx5_vdpa_virtqueue *mvq,
+ bool filled)
{
int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in);
u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {};
struct mlx5_vdpa_dev *mvdev = &ndev->mvdev;
struct mlx5_vdpa_mr *vq_mr;
struct mlx5_vdpa_mr *vq_desc_mr;
+ u64 features = filled ? mvdev->actual_features : mvdev->mlx_features;
void *obj_context;
u16 mlx_features;
void *cmd_hdr;
@@ -886,7 +902,7 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
goto err_alloc;
}
- mlx_features = get_features(ndev->mvdev.actual_features);
+ mlx_features = get_features(features);
cmd_hdr = MLX5_ADDR_OF(create_virtio_net_q_in, in, general_obj_in_cmd_hdr);
MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
@@ -894,8 +910,6 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid);
obj_context = MLX5_ADDR_OF(create_virtio_net_q_in, in, obj_context);
- MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx);
- MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx);
MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3,
mlx_features >> 3);
MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_2_0,
@@ -917,17 +931,36 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
MLX5_SET(virtio_q, vq_ctx, queue_index, mvq->index);
MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent);
MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0,
- !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1)));
- MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr);
- MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr);
- MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr);
- vq_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]];
- if (vq_mr)
- MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, vq_mr->mkey);
-
- vq_desc_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]];
- if (vq_desc_mr && MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, desc_group_mkey_supported))
- MLX5_SET(virtio_q, vq_ctx, desc_group_mkey, vq_desc_mr->mkey);
+ !!(features & BIT_ULL(VIRTIO_F_VERSION_1)));
+
+ if (filled) {
+ MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx);
+ MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx);
+
+ MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr);
+ MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr);
+ MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr);
+
+ vq_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]];
+ if (vq_mr)
+ MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, vq_mr->mkey);
+
+ vq_desc_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]];
+ if (vq_desc_mr &&
+ MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, desc_group_mkey_supported))
+ MLX5_SET(virtio_q, vq_ctx, desc_group_mkey, vq_desc_mr->mkey);
+ } else {
+ /* If there is no mr update, make sure that the existing ones are set
+ * modify to ready.
+ */
+ vq_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]];
+ if (vq_mr)
+ mvq->modified_fields |= MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY;
+
+ vq_desc_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]];
+ if (vq_desc_mr)
+ mvq->modified_fields |= MLX5_VIRTQ_MODIFY_MASK_DESC_GROUP_MKEY;
+ }
MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id);
MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size);
@@ -947,12 +980,15 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
kfree(in);
mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
- mlx5_vdpa_get_mr(mvdev, vq_mr);
- mvq->vq_mr = vq_mr;
+ if (filled) {
+ mlx5_vdpa_get_mr(mvdev, vq_mr);
+ mvq->vq_mr = vq_mr;
- if (vq_desc_mr && MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, desc_group_mkey_supported)) {
- mlx5_vdpa_get_mr(mvdev, vq_desc_mr);
- mvq->desc_mr = vq_desc_mr;
+ if (vq_desc_mr &&
+ MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, desc_group_mkey_supported)) {
+ mlx5_vdpa_get_mr(mvdev, vq_desc_mr);
+ mvq->desc_mr = vq_desc_mr;
+ }
}
return 0;
@@ -1273,6 +1309,19 @@ static int modify_virtqueue(struct mlx5_vdpa_net *ndev,
if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_USED_IDX)
MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx);
+ if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_QUEUE_VIRTIO_VERSION)
+ MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0,
+ !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1)));
+
+ if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_QUEUE_FEATURES) {
+ u16 mlx_features = get_features(ndev->mvdev.actual_features);
+
+ MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3,
+ mlx_features >> 3);
+ MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_2_0,
+ mlx_features & 7);
+ }
+
if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY) {
vq_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]];
@@ -1417,14 +1466,13 @@ static void dealloc_vector(struct mlx5_vdpa_net *ndev,
}
}
-static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
+static int setup_vq(struct mlx5_vdpa_net *ndev,
+ struct mlx5_vdpa_virtqueue *mvq,
+ bool filled)
{
u16 idx = mvq->index;
int err;
- if (!mvq->num_ent)
- return 0;
-
if (mvq->initialized)
return 0;
@@ -1449,20 +1497,18 @@ static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
goto err_connect;
alloc_vector(ndev, mvq);
- err = create_virtqueue(ndev, mvq);
+ err = create_virtqueue(ndev, mvq, filled);
if (err)
goto err_vq;
+ mvq->initialized = true;
+
if (mvq->ready) {
- err = modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY);
- if (err) {
- mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n",
- idx, err);
+ err = resume_vq(ndev, mvq);
+ if (err)
goto err_modify;
- }
}
- mvq->initialized = true;
return 0;
err_modify:
@@ -1479,51 +1525,105 @@ err_fwqp:
return err;
}
-static void suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
+static int suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
{
struct mlx5_virtq_attr attr;
+ int err;
if (!mvq->initialized)
- return;
+ return 0;
if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY)
- return;
+ return 0;
- if (modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND))
- mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed\n");
+ err = modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND);
+ if (err) {
+ mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed, err: %d\n", err);
+ return err;
+ }
- if (query_virtqueue(ndev, mvq, &attr)) {
- mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n");
- return;
+ err = query_virtqueue(ndev, mvq, &attr);
+ if (err) {
+ mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue, err: %d\n", err);
+ return err;
}
+
mvq->avail_idx = attr.available_index;
mvq->used_idx = attr.used_index;
+
+ return 0;
}
-static void suspend_vqs(struct mlx5_vdpa_net *ndev)
+static int suspend_vqs(struct mlx5_vdpa_net *ndev)
{
+ int err = 0;
int i;
- for (i = 0; i < ndev->mvdev.max_vqs; i++)
- suspend_vq(ndev, &ndev->vqs[i]);
+ for (i = 0; i < ndev->cur_num_vqs; i++) {
+ int local_err = suspend_vq(ndev, &ndev->vqs[i]);
+
+ err = local_err ? local_err : err;
+ }
+
+ return err;
}
-static void resume_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
+static int resume_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
{
- if (!mvq->initialized || !is_resumable(ndev))
- return;
+ int err;
- if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND)
- return;
+ if (!mvq->initialized)
+ return 0;
+
+ if (mvq->index >= ndev->cur_num_vqs)
+ return 0;
- if (modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY))
- mlx5_vdpa_warn(&ndev->mvdev, "modify to resume failed for vq %u\n", mvq->index);
+ switch (mvq->fw_state) {
+ case MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT:
+ /* Due to a FW quirk we need to modify the VQ fields first then change state.
+ * This should be fixed soon. After that, a single command can be used.
+ */
+ err = modify_virtqueue(ndev, mvq, 0);
+ if (err) {
+ mlx5_vdpa_warn(&ndev->mvdev,
+ "modify vq properties failed for vq %u, err: %d\n",
+ mvq->index, err);
+ return err;
+ }
+ break;
+ case MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND:
+ if (!is_resumable(ndev)) {
+ mlx5_vdpa_warn(&ndev->mvdev, "vq %d is not resumable\n", mvq->index);
+ return -EINVAL;
+ }
+ break;
+ case MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY:
+ return 0;
+ default:
+ mlx5_vdpa_warn(&ndev->mvdev, "resume vq %u called from bad state %d\n",
+ mvq->index, mvq->fw_state);
+ return -EINVAL;
+ }
+
+ err = modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY);
+ if (err)
+ mlx5_vdpa_warn(&ndev->mvdev, "modify to resume failed for vq %u, err: %d\n",
+ mvq->index, err);
+
+ return err;
}
-static void resume_vqs(struct mlx5_vdpa_net *ndev)
+static int resume_vqs(struct mlx5_vdpa_net *ndev)
{
- for (int i = 0; i < ndev->mvdev.max_vqs; i++)
- resume_vq(ndev, &ndev->vqs[i]);
+ int err = 0;
+
+ for (int i = 0; i < ndev->cur_num_vqs; i++) {
+ int local_err = resume_vq(ndev, &ndev->vqs[i]);
+
+ err = local_err ? local_err : err;
+ }
+
+ return err;
}
static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
@@ -2033,14 +2133,22 @@ static int change_num_qps(struct mlx5_vdpa_dev *mvdev, int newqps)
if (err)
return err;
- for (i = ndev->cur_num_vqs - 1; i >= 2 * newqps; i--)
- teardown_vq(ndev, &ndev->vqs[i]);
+ for (i = ndev->cur_num_vqs - 1; i >= 2 * newqps; i--) {
+ struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[i];
+
+ if (is_resumable(ndev))
+ suspend_vq(ndev, mvq);
+ else
+ teardown_vq(ndev, mvq);
+ }
ndev->cur_num_vqs = 2 * newqps;
} else {
ndev->cur_num_vqs = 2 * newqps;
for (i = cur_qps * 2; i < 2 * newqps; i++) {
- err = setup_vq(ndev, &ndev->vqs[i]);
+ struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[i];
+
+ err = mvq->initialized ? resume_vq(ndev, mvq) : setup_vq(ndev, mvq, true);
if (err)
goto clean_added;
}
@@ -2285,6 +2393,7 @@ static void mlx5_vdpa_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num)
}
mvq = &ndev->vqs[idx];
+ ndev->needs_teardown = num != mvq->num_ent;
mvq->num_ent = num;
}
@@ -2324,7 +2433,6 @@ static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready
struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
struct mlx5_vdpa_virtqueue *mvq;
- int err;
if (!mvdev->actual_features)
return;
@@ -2340,15 +2448,11 @@ static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready
mvq = &ndev->vqs[idx];
if (!ready) {
suspend_vq(ndev, mvq);
- } else {
- err = modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY);
- if (err) {
- mlx5_vdpa_warn(mvdev, "modify VQ %d to ready failed (%d)\n", idx, err);
+ } else if (mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK) {
+ if (resume_vq(ndev, mvq))
ready = false;
- }
}
-
mvq->ready = ready;
}
@@ -2536,14 +2640,14 @@ static int verify_driver_features(struct mlx5_vdpa_dev *mvdev, u64 features)
return 0;
}
-static int setup_virtqueues(struct mlx5_vdpa_dev *mvdev)
+static int setup_virtqueues(struct mlx5_vdpa_dev *mvdev, bool filled)
{
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
int err;
int i;
for (i = 0; i < mvdev->max_vqs; i++) {
- err = setup_vq(ndev, &ndev->vqs[i]);
+ err = setup_vq(ndev, &ndev->vqs[i], filled);
if (err)
goto err_vq;
}
@@ -2559,16 +2663,10 @@ err_vq:
static void teardown_virtqueues(struct mlx5_vdpa_net *ndev)
{
- struct mlx5_vdpa_virtqueue *mvq;
int i;
- for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) {
- mvq = &ndev->vqs[i];
- if (!mvq->initialized)
- continue;
-
- teardown_vq(ndev, mvq);
- }
+ for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--)
+ teardown_vq(ndev, &ndev->vqs[i]);
}
static void update_cvq_info(struct mlx5_vdpa_dev *mvdev)
@@ -2705,6 +2803,8 @@ static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
{
struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
+ u64 old_features = mvdev->actual_features;
+ u64 diff_features;
int err;
print_features(mvdev, features, true);
@@ -2714,20 +2814,26 @@ static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
return err;
ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features;
- if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ))
- ndev->rqt_size = mlx5vdpa16_to_cpu(mvdev, ndev->config.max_virtqueue_pairs);
- else
- ndev->rqt_size = 1;
- /* Device must start with 1 queue pair, as per VIRTIO v1.2 spec, section
- * 5.1.6.5.5 "Device operation in multiqueue mode":
- *
- * Multiqueue is disabled by default.
- * The driver enables multiqueue by sending a command using class
- * VIRTIO_NET_CTRL_MQ. The command selects the mode of multiqueue
- * operation, as follows: ...
- */
- ndev->cur_num_vqs = 2;
+ /* Interested in changes of vq features only. */
+ if (get_features(old_features) != get_features(mvdev->actual_features)) {
+ for (int i = 0; i < mvdev->max_vqs; ++i) {
+ struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[i];
+
+ mvq->modified_fields |= (
+ MLX5_VIRTQ_MODIFY_MASK_QUEUE_VIRTIO_VERSION |
+ MLX5_VIRTQ_MODIFY_MASK_QUEUE_FEATURES
+ );
+ }
+ }
+
+ /* When below features diverge from initial device features, VQs need a full teardown. */
+#define NEEDS_TEARDOWN_MASK (BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) | \
+ BIT_ULL(VIRTIO_NET_F_CSUM) | \
+ BIT_ULL(VIRTIO_F_VERSION_1))
+
+ diff_features = mvdev->mlx_features ^ mvdev->actual_features;
+ ndev->needs_teardown = !!(diff_features & NEEDS_TEARDOWN_MASK);
update_cvq_info(mvdev);
return err;
@@ -2816,7 +2922,7 @@ static void restore_channels_info(struct mlx5_vdpa_net *ndev)
int i;
mlx5_clear_vqs(ndev);
- init_mvqs(ndev);
+ mvqs_set_defaults(ndev);
for (i = 0; i < ndev->mvdev.max_vqs; i++) {
mvq = &ndev->vqs[i];
ri = &mvq->ri;
@@ -2848,12 +2954,12 @@ static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev,
if (err)
return err;
- teardown_driver(ndev);
+ teardown_vq_resources(ndev);
}
mlx5_vdpa_update_mr(mvdev, new_mr, asid);
- for (int i = 0; i < ndev->cur_num_vqs; i++)
+ for (int i = 0; i < mvdev->max_vqs; i++)
ndev->vqs[i].modified_fields |= MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY |
MLX5_VIRTQ_MODIFY_MASK_DESC_GROUP_MKEY;
@@ -2862,7 +2968,7 @@ static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev,
if (teardown) {
restore_channels_info(ndev);
- err = setup_driver(mvdev);
+ err = setup_vq_resources(ndev, true);
if (err)
return err;
}
@@ -2873,9 +2979,9 @@ static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev,
}
/* reslock must be held for this function */
-static int setup_driver(struct mlx5_vdpa_dev *mvdev)
+static int setup_vq_resources(struct mlx5_vdpa_net *ndev, bool filled)
{
- struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
+ struct mlx5_vdpa_dev *mvdev = &ndev->mvdev;
int err;
WARN_ON(!rwsem_is_locked(&ndev->reslock));
@@ -2891,7 +2997,7 @@ static int setup_driver(struct mlx5_vdpa_dev *mvdev)
if (err)
goto err_setup;
- err = setup_virtqueues(mvdev);
+ err = setup_virtqueues(mvdev, filled);
if (err) {
mlx5_vdpa_warn(mvdev, "setup_virtqueues\n");
goto err_setup;
@@ -2931,7 +3037,7 @@ out:
}
/* reslock must be held for this function */
-static void teardown_driver(struct mlx5_vdpa_net *ndev)
+static void teardown_vq_resources(struct mlx5_vdpa_net *ndev)
{
WARN_ON(!rwsem_is_locked(&ndev->reslock));
@@ -2945,18 +3051,7 @@ static void teardown_driver(struct mlx5_vdpa_net *ndev)
destroy_rqt(ndev);
teardown_virtqueues(ndev);
ndev->setup = false;
-}
-
-static void clear_vqs_ready(struct mlx5_vdpa_net *ndev)
-{
- int i;
-
- for (i = 0; i < ndev->mvdev.max_vqs; i++) {
- ndev->vqs[i].ready = false;
- ndev->vqs[i].modified_fields = 0;
- }
-
- ndev->mvdev.cvq.ready = false;
+ ndev->needs_teardown = false;
}
static int setup_cvq_vring(struct mlx5_vdpa_dev *mvdev)
@@ -2997,10 +3092,22 @@ static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status)
goto err_setup;
}
register_link_notifier(ndev);
- err = setup_driver(mvdev);
- if (err) {
- mlx5_vdpa_warn(mvdev, "failed to setup driver\n");
- goto err_driver;
+
+ if (ndev->needs_teardown)
+ teardown_vq_resources(ndev);
+
+ if (ndev->setup) {
+ err = resume_vqs(ndev);
+ if (err) {
+ mlx5_vdpa_warn(mvdev, "failed to resume VQs\n");
+ goto err_driver;
+ }
+ } else {
+ err = setup_vq_resources(ndev, true);
+ if (err) {
+ mlx5_vdpa_warn(mvdev, "failed to setup driver\n");
+ goto err_driver;
+ }
}
} else {
mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n");
@@ -3030,23 +3137,48 @@ static void init_group_to_asid_map(struct mlx5_vdpa_dev *mvdev)
mvdev->group2asid[i] = 0;
}
+static bool needs_vqs_reset(const struct mlx5_vdpa_dev *mvdev)
+{
+ struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
+ struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[0];
+
+ if (mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)
+ return true;
+
+ if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT)
+ return true;
+
+ return mvq->modified_fields & (
+ MLX5_VIRTQ_MODIFY_MASK_STATE |
+ MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_ADDRS |
+ MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_AVAIL_IDX |
+ MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_USED_IDX
+ );
+}
+
static int mlx5_vdpa_compat_reset(struct vdpa_device *vdev, u32 flags)
{
struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
+ bool vq_reset;
print_status(mvdev, 0, true);
mlx5_vdpa_info(mvdev, "performing device reset\n");
down_write(&ndev->reslock);
unregister_link_notifier(ndev);
- teardown_driver(ndev);
- clear_vqs_ready(ndev);
+ vq_reset = needs_vqs_reset(mvdev);
+ if (vq_reset) {
+ teardown_vq_resources(ndev);
+ mvqs_set_defaults(ndev);
+ }
+
if (flags & VDPA_RESET_F_CLEAN_MAP)
mlx5_vdpa_destroy_mr_resources(&ndev->mvdev);
ndev->mvdev.status = 0;
ndev->mvdev.suspended = false;
- ndev->cur_num_vqs = 0;
+ ndev->cur_num_vqs = MLX5V_DEFAULT_VQ_COUNT;
+ ndev->mvdev.cvq.ready = false;
ndev->mvdev.cvq.received_desc = 0;
ndev->mvdev.cvq.completed_desc = 0;
memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1));
@@ -3059,6 +3191,8 @@ static int mlx5_vdpa_compat_reset(struct vdpa_device *vdev, u32 flags)
if (mlx5_vdpa_create_dma_mr(mvdev))
mlx5_vdpa_warn(mvdev, "create MR failed\n");
}
+ if (vq_reset)
+ setup_vq_resources(ndev, false);
up_write(&ndev->reslock);
return 0;
@@ -3197,7 +3331,7 @@ static void mlx5_vdpa_free(struct vdpa_device *vdev)
ndev = to_mlx5_vdpa_ndev(mvdev);
- free_resources(ndev);
+ free_fixed_resources(ndev);
mlx5_vdpa_destroy_mr_resources(mvdev);
if (!is_zero_ether_addr(ndev->config.mac)) {
pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev));
@@ -3361,27 +3495,25 @@ static int mlx5_vdpa_suspend(struct vdpa_device *vdev)
{
struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
- struct mlx5_vdpa_virtqueue *mvq;
- int i;
+ int err;
mlx5_vdpa_info(mvdev, "suspending device\n");
down_write(&ndev->reslock);
unregister_link_notifier(ndev);
- for (i = 0; i < ndev->cur_num_vqs; i++) {
- mvq = &ndev->vqs[i];
- suspend_vq(ndev, mvq);
- }
+ err = suspend_vqs(ndev);
mlx5_vdpa_cvq_suspend(mvdev);
mvdev->suspended = true;
up_write(&ndev->reslock);
- return 0;
+
+ return err;
}
static int mlx5_vdpa_resume(struct vdpa_device *vdev)
{
struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
struct mlx5_vdpa_net *ndev;
+ int err;
ndev = to_mlx5_vdpa_ndev(mvdev);
@@ -3389,10 +3521,11 @@ static int mlx5_vdpa_resume(struct vdpa_device *vdev)
down_write(&ndev->reslock);
mvdev->suspended = false;
- resume_vqs(ndev);
+ err = resume_vqs(ndev);
register_link_notifier(ndev);
up_write(&ndev->reslock);
- return 0;
+
+ return err;
}
static int mlx5_set_group_asid(struct vdpa_device *vdev, u32 group,
@@ -3467,7 +3600,7 @@ static int query_mtu(struct mlx5_core_dev *mdev, u16 *mtu)
return 0;
}
-static int alloc_resources(struct mlx5_vdpa_net *ndev)
+static int alloc_fixed_resources(struct mlx5_vdpa_net *ndev)
{
struct mlx5_vdpa_net_resources *res = &ndev->res;
int err;
@@ -3494,7 +3627,7 @@ err_tis:
return err;
}
-static void free_resources(struct mlx5_vdpa_net *ndev)
+static void free_fixed_resources(struct mlx5_vdpa_net *ndev)
{
struct mlx5_vdpa_net_resources *res = &ndev->res;
@@ -3506,7 +3639,7 @@ static void free_resources(struct mlx5_vdpa_net *ndev)
res->valid = false;
}
-static void init_mvqs(struct mlx5_vdpa_net *ndev)
+static void mvqs_set_defaults(struct mlx5_vdpa_net *ndev)
{
struct mlx5_vdpa_virtqueue *mvq;
int i;
@@ -3518,12 +3651,7 @@ static void init_mvqs(struct mlx5_vdpa_net *ndev)
mvq->ndev = ndev;
mvq->fwqp.fw = true;
mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE;
- }
- for (; i < ndev->mvdev.max_vqs; i++) {
- mvq = &ndev->vqs[i];
- memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri));
- mvq->index = i;
- mvq->ndev = ndev;
+ mvq->num_ent = MLX5V_DEFAULT_VQ_SIZE;
}
}
@@ -3660,8 +3788,9 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
err = -ENOMEM;
goto err_alloc;
}
+ ndev->cur_num_vqs = MLX5V_DEFAULT_VQ_COUNT;
- init_mvqs(ndev);
+ mvqs_set_defaults(ndev);
allocate_irqs(ndev);
init_rwsem(&ndev->reslock);
config = &ndev->config;
@@ -3718,8 +3847,12 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
goto err_alloc;
}
- if (device_features & BIT_ULL(VIRTIO_NET_F_MQ))
+ if (device_features & BIT_ULL(VIRTIO_NET_F_MQ)) {
config->max_virtqueue_pairs = cpu_to_mlx5vdpa16(mvdev, max_vqs / 2);
+ ndev->rqt_size = max_vqs / 2;
+ } else {
+ ndev->rqt_size = 1;
+ }
ndev->mvdev.mlx_features = device_features;
mvdev->vdev.dma_dev = &mdev->pdev->dev;
@@ -3735,7 +3868,7 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
goto err_res;
}
- err = alloc_resources(ndev);
+ err = alloc_fixed_resources(ndev);
if (err)
goto err_mr;
@@ -3753,12 +3886,25 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
goto err_reg;
mgtdev->ndev = ndev;
+
+ /* For virtio-vdpa, the device was set up during device register. */
+ if (ndev->setup)
+ return 0;
+
+ down_write(&ndev->reslock);
+ err = setup_vq_resources(ndev, false);
+ up_write(&ndev->reslock);
+ if (err)
+ goto err_setup_vq_res;
+
return 0;
+err_setup_vq_res:
+ _vdpa_unregister_device(&mvdev->vdev);
err_reg:
destroy_workqueue(mvdev->wq);
err_res2:
- free_resources(ndev);
+ free_fixed_resources(ndev);
err_mr:
mlx5_vdpa_destroy_mr_resources(mvdev);
err_res:
@@ -3780,6 +3926,11 @@ static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *
unregister_link_notifier(ndev);
_vdpa_unregister_device(dev);
+
+ down_write(&ndev->reslock);
+ teardown_vq_resources(ndev);
+ up_write(&ndev->reslock);
+
wq = mvdev->wq;
mvdev->wq = NULL;
destroy_workqueue(wq);
diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.h b/drivers/vdpa/mlx5/net/mlx5_vnet.h
index 90b556a57971..00e79a7d0be8 100644
--- a/drivers/vdpa/mlx5/net/mlx5_vnet.h
+++ b/drivers/vdpa/mlx5/net/mlx5_vnet.h
@@ -56,6 +56,7 @@ struct mlx5_vdpa_net {
struct dentry *rx_dent;
struct dentry *rx_table_dent;
bool setup;
+ bool needs_teardown;
u32 cur_num_vqs;
u32 rqt_size;
bool nb_registered;
diff --git a/drivers/vdpa/octeon_ep/Makefile b/drivers/vdpa/octeon_ep/Makefile
new file mode 100644
index 000000000000..e23e2ff14f33
--- /dev/null
+++ b/drivers/vdpa/octeon_ep/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_OCTEONEP_VDPA) += octep_vdpa.o
+octep_vdpa-$(CONFIG_OCTEONEP_VDPA) += octep_vdpa_main.o
+octep_vdpa-$(CONFIG_OCTEONEP_VDPA) += octep_vdpa_hw.o
diff --git a/drivers/vdpa/octeon_ep/octep_vdpa.h b/drivers/vdpa/octeon_ep/octep_vdpa.h
new file mode 100644
index 000000000000..046710ec4d42
--- /dev/null
+++ b/drivers/vdpa/octeon_ep/octep_vdpa.h
@@ -0,0 +1,94 @@
+/* SPDX-License-Identifier: GPL-2.0-only
+ * Copyright (C) 2024 Marvell.
+ */
+#ifndef __OCTEP_VDPA_H__
+#define __OCTEP_VDPA_H__
+
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <linux/vdpa.h>
+#include <linux/virtio_pci_modern.h>
+#include <uapi/linux/virtio_net.h>
+#include <uapi/linux/virtio_blk.h>
+#include <uapi/linux/virtio_config.h>
+#include <uapi/linux/virtio_pci.h>
+#include <uapi/linux/vdpa.h>
+
+#define OCTEP_VDPA_DEVID_CN106K_PF 0xb900
+#define OCTEP_VDPA_DEVID_CN106K_VF 0xb903
+#define OCTEP_VDPA_DEVID_CN105K_PF 0xba00
+#define OCTEP_VDPA_DEVID_CN105K_VF 0xba03
+#define OCTEP_VDPA_DEVID_CN103K_PF 0xbd00
+#define OCTEP_VDPA_DEVID_CN103K_VF 0xbd03
+
+#define OCTEP_HW_MBOX_BAR 0
+#define OCTEP_HW_CAPS_BAR 4
+
+#define OCTEP_DEV_READY_SIGNATURE 0xBABABABA
+
+#define OCTEP_EPF_RINFO(x) (0x000209f0 | ((x) << 25))
+#define OCTEP_VF_MBOX_DATA(x) (0x00010210 | ((x) << 17))
+#define OCTEP_PF_MBOX_DATA(x) (0x00022000 | ((x) << 4))
+
+#define OCTEP_EPF_RINFO_RPVF(val) (((val) >> 32) & 0xF)
+#define OCTEP_EPF_RINFO_NVFS(val) (((val) >> 48) & 0x7F)
+
+#define OCTEP_FW_READY_SIGNATURE0 0xFEEDFEED
+#define OCTEP_FW_READY_SIGNATURE1 0x3355ffaa
+
+enum octep_vdpa_dev_status {
+ OCTEP_VDPA_DEV_STATUS_INVALID,
+ OCTEP_VDPA_DEV_STATUS_ALLOC,
+ OCTEP_VDPA_DEV_STATUS_WAIT_FOR_BAR_INIT,
+ OCTEP_VDPA_DEV_STATUS_INIT,
+ OCTEP_VDPA_DEV_STATUS_READY,
+ OCTEP_VDPA_DEV_STATUS_UNINIT
+};
+
+struct octep_vring_info {
+ struct vdpa_callback cb;
+ void __iomem *notify_addr;
+ u32 __iomem *cb_notify_addr;
+ phys_addr_t notify_pa;
+ char msix_name[256];
+};
+
+struct octep_hw {
+ struct pci_dev *pdev;
+ u8 __iomem *base[PCI_STD_NUM_BARS];
+ struct virtio_pci_common_cfg __iomem *common_cfg;
+ u8 __iomem *dev_cfg;
+ u8 __iomem *isr;
+ void __iomem *notify_base;
+ phys_addr_t notify_base_pa;
+ u32 notify_off_multiplier;
+ u8 notify_bar;
+ struct octep_vring_info *vqs;
+ struct vdpa_callback config_cb;
+ u64 features;
+ u16 nr_vring;
+ u32 config_size;
+ int irq;
+};
+
+u8 octep_hw_get_status(struct octep_hw *oct_hw);
+void octep_hw_set_status(struct octep_hw *dev, uint8_t status);
+void octep_hw_reset(struct octep_hw *oct_hw);
+void octep_write_queue_select(struct octep_hw *oct_hw, u16 queue_id);
+void octep_notify_queue(struct octep_hw *oct_hw, u16 qid);
+void octep_read_dev_config(struct octep_hw *oct_hw, u64 offset, void *dst, int length);
+int octep_set_vq_address(struct octep_hw *oct_hw, u16 qid, u64 desc_area, u64 driver_area,
+ u64 device_area);
+void octep_set_vq_num(struct octep_hw *oct_hw, u16 qid, u32 num);
+void octep_set_vq_ready(struct octep_hw *oct_hw, u16 qid, bool ready);
+bool octep_get_vq_ready(struct octep_hw *oct_hw, u16 qid);
+int octep_set_vq_state(struct octep_hw *oct_hw, u16 qid, const struct vdpa_vq_state *state);
+int octep_get_vq_state(struct octep_hw *oct_hw, u16 qid, struct vdpa_vq_state *state);
+u16 octep_get_vq_size(struct octep_hw *oct_hw);
+int octep_hw_caps_read(struct octep_hw *oct_hw, struct pci_dev *pdev);
+u64 octep_hw_get_dev_features(struct octep_hw *oct_hw);
+void octep_hw_set_drv_features(struct octep_hw *oct_hw, u64 features);
+u64 octep_hw_get_drv_features(struct octep_hw *oct_hw);
+int octep_verify_features(u64 features);
+
+#endif /* __OCTEP_VDPA_H__ */
diff --git a/drivers/vdpa/octeon_ep/octep_vdpa_hw.c b/drivers/vdpa/octeon_ep/octep_vdpa_hw.c
new file mode 100644
index 000000000000..7fa0491bb201
--- /dev/null
+++ b/drivers/vdpa/octeon_ep/octep_vdpa_hw.c
@@ -0,0 +1,517 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (C) 2024 Marvell. */
+
+#include <linux/iopoll.h>
+
+#include "octep_vdpa.h"
+
+enum octep_mbox_ids {
+ OCTEP_MBOX_MSG_SET_VQ_STATE = 1,
+ OCTEP_MBOX_MSG_GET_VQ_STATE,
+};
+
+#define OCTEP_HW_TIMEOUT 10000000
+
+#define MBOX_OFFSET 64
+#define MBOX_RSP_MASK 0x00000001
+#define MBOX_RC_MASK 0x0000FFFE
+
+#define MBOX_RSP_TO_ERR(val) (-(((val) & MBOX_RC_MASK) >> 2))
+#define MBOX_AVAIL(val) (((val) & MBOX_RSP_MASK))
+#define MBOX_RSP(val) ((val) & (MBOX_RC_MASK | MBOX_RSP_MASK))
+
+#define DEV_RST_ACK_BIT 7
+#define FEATURE_SEL_ACK_BIT 15
+#define QUEUE_SEL_ACK_BIT 15
+
+struct octep_mbox_hdr {
+ u8 ver;
+ u8 rsvd1;
+ u16 id;
+ u16 rsvd2;
+#define MBOX_REQ_SIG (0xdead)
+#define MBOX_RSP_SIG (0xbeef)
+ u16 sig;
+};
+
+struct octep_mbox_sts {
+ u16 rsp:1;
+ u16 rc:15;
+ u16 rsvd;
+};
+
+struct octep_mbox {
+ struct octep_mbox_hdr hdr;
+ struct octep_mbox_sts sts;
+ u64 rsvd;
+ u32 data[];
+};
+
+static inline struct octep_mbox __iomem *octep_get_mbox(struct octep_hw *oct_hw)
+{
+ return (struct octep_mbox __iomem *)(oct_hw->dev_cfg + MBOX_OFFSET);
+}
+
+static inline int octep_wait_for_mbox_avail(struct octep_mbox __iomem *mbox)
+{
+ u32 val;
+
+ return readx_poll_timeout(ioread32, &mbox->sts, val, MBOX_AVAIL(val), 10,
+ OCTEP_HW_TIMEOUT);
+}
+
+static inline int octep_wait_for_mbox_rsp(struct octep_mbox __iomem *mbox)
+{
+ u32 val;
+
+ return readx_poll_timeout(ioread32, &mbox->sts, val, MBOX_RSP(val), 10,
+ OCTEP_HW_TIMEOUT);
+}
+
+static inline void octep_write_hdr(struct octep_mbox __iomem *mbox, u16 id, u16 sig)
+{
+ iowrite16(id, &mbox->hdr.id);
+ iowrite16(sig, &mbox->hdr.sig);
+}
+
+static inline u32 octep_read_sig(struct octep_mbox __iomem *mbox)
+{
+ return ioread16(&mbox->hdr.sig);
+}
+
+static inline void octep_write_sts(struct octep_mbox __iomem *mbox, u32 sts)
+{
+ iowrite32(sts, &mbox->sts);
+}
+
+static inline u32 octep_read_sts(struct octep_mbox __iomem *mbox)
+{
+ return ioread32(&mbox->sts);
+}
+
+static inline u32 octep_read32_word(struct octep_mbox __iomem *mbox, u16 word_idx)
+{
+ return ioread32(&mbox->data[word_idx]);
+}
+
+static inline void octep_write32_word(struct octep_mbox __iomem *mbox, u16 word_idx, u32 word)
+{
+ return iowrite32(word, &mbox->data[word_idx]);
+}
+
+static int octep_process_mbox(struct octep_hw *oct_hw, u16 id, u16 qid, void *buffer,
+ u32 buf_size, bool write)
+{
+ struct octep_mbox __iomem *mbox = octep_get_mbox(oct_hw);
+ struct pci_dev *pdev = oct_hw->pdev;
+ u32 *p = (u32 *)buffer;
+ u16 data_wds;
+ int ret, i;
+ u32 val;
+
+ if (!IS_ALIGNED(buf_size, 4))
+ return -EINVAL;
+
+ /* Make sure mbox space is available */
+ ret = octep_wait_for_mbox_avail(mbox);
+ if (ret) {
+ dev_warn(&pdev->dev, "Timeout waiting for previous mbox data to be consumed\n");
+ return ret;
+ }
+ data_wds = buf_size / 4;
+
+ if (write) {
+ for (i = 1; i <= data_wds; i++) {
+ octep_write32_word(mbox, i, *p);
+ p++;
+ }
+ }
+ octep_write32_word(mbox, 0, (u32)qid);
+ octep_write_sts(mbox, 0);
+
+ octep_write_hdr(mbox, id, MBOX_REQ_SIG);
+
+ ret = octep_wait_for_mbox_rsp(mbox);
+ if (ret) {
+ dev_warn(&pdev->dev, "Timeout waiting for mbox : %d response\n", id);
+ return ret;
+ }
+
+ val = octep_read_sig(mbox);
+ if ((val & 0xFFFF) != MBOX_RSP_SIG) {
+ dev_warn(&pdev->dev, "Invalid Signature from mbox : %d response\n", id);
+ return ret;
+ }
+
+ val = octep_read_sts(mbox);
+ if (val & MBOX_RC_MASK) {
+ ret = MBOX_RSP_TO_ERR(val);
+ dev_warn(&pdev->dev, "Error while processing mbox : %d, err %d\n", id, ret);
+ return ret;
+ }
+
+ if (!write)
+ for (i = 1; i <= data_wds; i++)
+ *p++ = octep_read32_word(mbox, i);
+
+ return 0;
+}
+
+static void octep_mbox_init(struct octep_mbox __iomem *mbox)
+{
+ iowrite32(1, &mbox->sts);
+}
+
+int octep_verify_features(u64 features)
+{
+ /* Minimum features to expect */
+ if (!(features & BIT_ULL(VIRTIO_F_VERSION_1)))
+ return -EOPNOTSUPP;
+
+ if (!(features & BIT_ULL(VIRTIO_F_NOTIFICATION_DATA)))
+ return -EOPNOTSUPP;
+
+ if (!(features & BIT_ULL(VIRTIO_F_RING_PACKED)))
+ return -EOPNOTSUPP;
+
+ return 0;
+}
+
+u8 octep_hw_get_status(struct octep_hw *oct_hw)
+{
+ return ioread8(&oct_hw->common_cfg->device_status);
+}
+
+void octep_hw_set_status(struct octep_hw *oct_hw, u8 status)
+{
+ iowrite8(status, &oct_hw->common_cfg->device_status);
+}
+
+void octep_hw_reset(struct octep_hw *oct_hw)
+{
+ u8 val;
+
+ octep_hw_set_status(oct_hw, 0 | BIT(DEV_RST_ACK_BIT));
+ if (readx_poll_timeout(ioread8, &oct_hw->common_cfg->device_status, val, !val, 10,
+ OCTEP_HW_TIMEOUT)) {
+ dev_warn(&oct_hw->pdev->dev, "Octeon device reset timeout\n");
+ return;
+ }
+}
+
+static int feature_sel_write_with_timeout(struct octep_hw *oct_hw, u32 select, void __iomem *addr)
+{
+ u32 val;
+
+ iowrite32(select | BIT(FEATURE_SEL_ACK_BIT), addr);
+
+ if (readx_poll_timeout(ioread32, addr, val, val == select, 10, OCTEP_HW_TIMEOUT)) {
+ dev_warn(&oct_hw->pdev->dev, "Feature select%d write timeout\n", select);
+ return -1;
+ }
+ return 0;
+}
+
+u64 octep_hw_get_dev_features(struct octep_hw *oct_hw)
+{
+ u32 features_lo, features_hi;
+
+ if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->device_feature_select))
+ return 0;
+
+ features_lo = ioread32(&oct_hw->common_cfg->device_feature);
+
+ if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->device_feature_select))
+ return 0;
+
+ features_hi = ioread32(&oct_hw->common_cfg->device_feature);
+
+ return ((u64)features_hi << 32) | features_lo;
+}
+
+u64 octep_hw_get_drv_features(struct octep_hw *oct_hw)
+{
+ u32 features_lo, features_hi;
+
+ if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->guest_feature_select))
+ return 0;
+
+ features_lo = ioread32(&oct_hw->common_cfg->guest_feature);
+
+ if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->guest_feature_select))
+ return 0;
+
+ features_hi = ioread32(&oct_hw->common_cfg->guest_feature);
+
+ return ((u64)features_hi << 32) | features_lo;
+}
+
+void octep_hw_set_drv_features(struct octep_hw *oct_hw, u64 features)
+{
+ if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->guest_feature_select))
+ return;
+
+ iowrite32(features & (BIT_ULL(32) - 1), &oct_hw->common_cfg->guest_feature);
+
+ if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->guest_feature_select))
+ return;
+
+ iowrite32(features >> 32, &oct_hw->common_cfg->guest_feature);
+}
+
+void octep_write_queue_select(struct octep_hw *oct_hw, u16 queue_id)
+{
+ u16 val;
+
+ iowrite16(queue_id | BIT(QUEUE_SEL_ACK_BIT), &oct_hw->common_cfg->queue_select);
+
+ if (readx_poll_timeout(ioread16, &oct_hw->common_cfg->queue_select, val, val == queue_id,
+ 10, OCTEP_HW_TIMEOUT)) {
+ dev_warn(&oct_hw->pdev->dev, "Queue select write timeout\n");
+ return;
+ }
+}
+
+void octep_notify_queue(struct octep_hw *oct_hw, u16 qid)
+{
+ iowrite16(qid, oct_hw->vqs[qid].notify_addr);
+}
+
+void octep_read_dev_config(struct octep_hw *oct_hw, u64 offset, void *dst, int length)
+{
+ u8 old_gen, new_gen, *p;
+ int i;
+
+ if (WARN_ON(offset + length > oct_hw->config_size))
+ return;
+
+ do {
+ old_gen = ioread8(&oct_hw->common_cfg->config_generation);
+ p = dst;
+ for (i = 0; i < length; i++)
+ *p++ = ioread8(oct_hw->dev_cfg + offset + i);
+
+ new_gen = ioread8(&oct_hw->common_cfg->config_generation);
+ } while (old_gen != new_gen);
+}
+
+int octep_set_vq_address(struct octep_hw *oct_hw, u16 qid, u64 desc_area, u64 driver_area,
+ u64 device_area)
+{
+ struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;
+
+ octep_write_queue_select(oct_hw, qid);
+ vp_iowrite64_twopart(desc_area, &cfg->queue_desc_lo,
+ &cfg->queue_desc_hi);
+ vp_iowrite64_twopart(driver_area, &cfg->queue_avail_lo,
+ &cfg->queue_avail_hi);
+ vp_iowrite64_twopart(device_area, &cfg->queue_used_lo,
+ &cfg->queue_used_hi);
+
+ return 0;
+}
+
+int octep_get_vq_state(struct octep_hw *oct_hw, u16 qid, struct vdpa_vq_state *state)
+{
+ return octep_process_mbox(oct_hw, OCTEP_MBOX_MSG_GET_VQ_STATE, qid, state,
+ sizeof(*state), 0);
+}
+
+int octep_set_vq_state(struct octep_hw *oct_hw, u16 qid, const struct vdpa_vq_state *state)
+{
+ struct vdpa_vq_state q_state;
+
+ memcpy(&q_state, state, sizeof(struct vdpa_vq_state));
+ return octep_process_mbox(oct_hw, OCTEP_MBOX_MSG_SET_VQ_STATE, qid, &q_state,
+ sizeof(*state), 1);
+}
+
+void octep_set_vq_num(struct octep_hw *oct_hw, u16 qid, u32 num)
+{
+ struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;
+
+ octep_write_queue_select(oct_hw, qid);
+ iowrite16(num, &cfg->queue_size);
+}
+
+void octep_set_vq_ready(struct octep_hw *oct_hw, u16 qid, bool ready)
+{
+ struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;
+
+ octep_write_queue_select(oct_hw, qid);
+ iowrite16(ready, &cfg->queue_enable);
+}
+
+bool octep_get_vq_ready(struct octep_hw *oct_hw, u16 qid)
+{
+ struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;
+
+ octep_write_queue_select(oct_hw, qid);
+ return ioread16(&cfg->queue_enable);
+}
+
+u16 octep_get_vq_size(struct octep_hw *oct_hw)
+{
+ octep_write_queue_select(oct_hw, 0);
+ return ioread16(&oct_hw->common_cfg->queue_size);
+}
+
+static u32 octep_get_config_size(struct octep_hw *oct_hw)
+{
+ return sizeof(struct virtio_net_config);
+}
+
+static void __iomem *octep_get_cap_addr(struct octep_hw *oct_hw, struct virtio_pci_cap *cap)
+{
+ struct device *dev = &oct_hw->pdev->dev;
+ u32 length = le32_to_cpu(cap->length);
+ u32 offset = le32_to_cpu(cap->offset);
+ u8 bar = cap->bar;
+ u32 len;
+
+ if (bar != OCTEP_HW_CAPS_BAR) {
+ dev_err(dev, "Invalid bar: %u\n", bar);
+ return NULL;
+ }
+ if (offset + length < offset) {
+ dev_err(dev, "offset(%u) + length(%u) overflows\n",
+ offset, length);
+ return NULL;
+ }
+ len = pci_resource_len(oct_hw->pdev, bar);
+ if (offset + length > len) {
+ dev_err(dev, "invalid cap: overflows bar space: %u > %u\n",
+ offset + length, len);
+ return NULL;
+ }
+ return oct_hw->base[bar] + offset;
+}
+
+/* In Octeon DPU device, the virtio config space is completely
+ * emulated by the device's firmware. So, the standard pci config
+ * read apis can't be used for reading the virtio capability.
+ */
+static void octep_pci_caps_read(struct octep_hw *oct_hw, void *buf, size_t len, off_t offset)
+{
+ u8 __iomem *bar = oct_hw->base[OCTEP_HW_CAPS_BAR];
+ u8 *p = buf;
+ size_t i;
+
+ for (i = 0; i < len; i++)
+ *p++ = ioread8(bar + offset + i);
+}
+
+static int octep_pci_signature_verify(struct octep_hw *oct_hw)
+{
+ u32 signature[2];
+
+ octep_pci_caps_read(oct_hw, &signature, sizeof(signature), 0);
+
+ if (signature[0] != OCTEP_FW_READY_SIGNATURE0)
+ return -1;
+
+ if (signature[1] != OCTEP_FW_READY_SIGNATURE1)
+ return -1;
+
+ return 0;
+}
+
+int octep_hw_caps_read(struct octep_hw *oct_hw, struct pci_dev *pdev)
+{
+ struct octep_mbox __iomem *mbox;
+ struct device *dev = &pdev->dev;
+ struct virtio_pci_cap cap;
+ u16 notify_off;
+ int i, ret;
+ u8 pos;
+
+ oct_hw->pdev = pdev;
+ ret = octep_pci_signature_verify(oct_hw);
+ if (ret) {
+ dev_err(dev, "Octeon Virtio FW is not initialized\n");
+ return -EIO;
+ }
+
+ octep_pci_caps_read(oct_hw, &pos, 1, PCI_CAPABILITY_LIST);
+
+ while (pos) {
+ octep_pci_caps_read(oct_hw, &cap, 2, pos);
+
+ if (cap.cap_vndr != PCI_CAP_ID_VNDR) {
+ dev_err(dev, "Found invalid capability vndr id: %d\n", cap.cap_vndr);
+ break;
+ }
+
+ octep_pci_caps_read(oct_hw, &cap, sizeof(cap), pos);
+
+ dev_info(dev, "[%2x] cfg type: %u, bar: %u, offset: %04x, len: %u\n",
+ pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
+
+ switch (cap.cfg_type) {
+ case VIRTIO_PCI_CAP_COMMON_CFG:
+ oct_hw->common_cfg = octep_get_cap_addr(oct_hw, &cap);
+ break;
+ case VIRTIO_PCI_CAP_NOTIFY_CFG:
+ octep_pci_caps_read(oct_hw, &oct_hw->notify_off_multiplier,
+ 4, pos + sizeof(cap));
+
+ oct_hw->notify_base = octep_get_cap_addr(oct_hw, &cap);
+ oct_hw->notify_bar = cap.bar;
+ oct_hw->notify_base_pa = pci_resource_start(pdev, cap.bar) +
+ le32_to_cpu(cap.offset);
+ break;
+ case VIRTIO_PCI_CAP_DEVICE_CFG:
+ oct_hw->dev_cfg = octep_get_cap_addr(oct_hw, &cap);
+ break;
+ case VIRTIO_PCI_CAP_ISR_CFG:
+ oct_hw->isr = octep_get_cap_addr(oct_hw, &cap);
+ break;
+ }
+
+ pos = cap.cap_next;
+ }
+ if (!oct_hw->common_cfg || !oct_hw->notify_base ||
+ !oct_hw->dev_cfg || !oct_hw->isr) {
+ dev_err(dev, "Incomplete PCI capabilities");
+ return -EIO;
+ }
+ dev_info(dev, "common cfg mapped at: 0x%016llx\n", (u64)(uintptr_t)oct_hw->common_cfg);
+ dev_info(dev, "device cfg mapped at: 0x%016llx\n", (u64)(uintptr_t)oct_hw->dev_cfg);
+ dev_info(dev, "isr cfg mapped at: 0x%016llx\n", (u64)(uintptr_t)oct_hw->isr);
+ dev_info(dev, "notify base: 0x%016llx, notify off multiplier: %u\n",
+ (u64)(uintptr_t)oct_hw->notify_base, oct_hw->notify_off_multiplier);
+
+ oct_hw->config_size = octep_get_config_size(oct_hw);
+ oct_hw->features = octep_hw_get_dev_features(oct_hw);
+
+ ret = octep_verify_features(oct_hw->features);
+ if (ret) {
+ dev_err(&pdev->dev, "Couldn't read features from the device FW\n");
+ return ret;
+ }
+ oct_hw->nr_vring = vp_ioread16(&oct_hw->common_cfg->num_queues);
+
+ oct_hw->vqs = devm_kcalloc(&pdev->dev, oct_hw->nr_vring, sizeof(*oct_hw->vqs), GFP_KERNEL);
+ if (!oct_hw->vqs)
+ return -ENOMEM;
+
+ oct_hw->irq = -1;
+
+ dev_info(&pdev->dev, "Device features : %llx\n", oct_hw->features);
+ dev_info(&pdev->dev, "Maximum queues : %u\n", oct_hw->nr_vring);
+
+ for (i = 0; i < oct_hw->nr_vring; i++) {
+ octep_write_queue_select(oct_hw, i);
+ notify_off = vp_ioread16(&oct_hw->common_cfg->queue_notify_off);
+ oct_hw->vqs[i].notify_addr = oct_hw->notify_base +
+ notify_off * oct_hw->notify_off_multiplier;
+ oct_hw->vqs[i].cb_notify_addr = (u32 __iomem *)oct_hw->vqs[i].notify_addr + 1;
+ oct_hw->vqs[i].notify_pa = oct_hw->notify_base_pa +
+ notify_off * oct_hw->notify_off_multiplier;
+ }
+ mbox = octep_get_mbox(oct_hw);
+ octep_mbox_init(mbox);
+ dev_info(dev, "mbox mapped at: 0x%016llx\n", (u64)(uintptr_t)mbox);
+
+ return 0;
+}
diff --git a/drivers/vdpa/octeon_ep/octep_vdpa_main.c b/drivers/vdpa/octeon_ep/octep_vdpa_main.c
new file mode 100644
index 000000000000..cd55b1aac151
--- /dev/null
+++ b/drivers/vdpa/octeon_ep/octep_vdpa_main.c
@@ -0,0 +1,857 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (C) 2024 Marvell. */
+
+#include <linux/interrupt.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/module.h>
+#include <linux/iommu.h>
+#include "octep_vdpa.h"
+
+#define OCTEP_VDPA_DRIVER_NAME "octep_vdpa"
+
+struct octep_pf {
+ u8 __iomem *base[PCI_STD_NUM_BARS];
+ struct pci_dev *pdev;
+ struct resource res;
+ u64 vf_base;
+ int enabled_vfs;
+ u32 vf_stride;
+ u16 vf_devid;
+};
+
+struct octep_vdpa {
+ struct vdpa_device vdpa;
+ struct octep_hw *oct_hw;
+ struct pci_dev *pdev;
+};
+
+struct octep_vdpa_mgmt_dev {
+ struct vdpa_mgmt_dev mdev;
+ struct octep_hw oct_hw;
+ struct pci_dev *pdev;
+ /* Work entry to handle device setup */
+ struct work_struct setup_task;
+ /* Device status */
+ atomic_t status;
+};
+
+static struct octep_hw *vdpa_to_octep_hw(struct vdpa_device *vdpa_dev)
+{
+ struct octep_vdpa *oct_vdpa;
+
+ oct_vdpa = container_of(vdpa_dev, struct octep_vdpa, vdpa);
+
+ return oct_vdpa->oct_hw;
+}
+
+static irqreturn_t octep_vdpa_intr_handler(int irq, void *data)
+{
+ struct octep_hw *oct_hw = data;
+ int i;
+
+ for (i = 0; i < oct_hw->nr_vring; i++) {
+ if (oct_hw->vqs[i].cb.callback && ioread32(oct_hw->vqs[i].cb_notify_addr)) {
+ /* Acknowledge the per queue notification to the device */
+ iowrite32(0, oct_hw->vqs[i].cb_notify_addr);
+ oct_hw->vqs[i].cb.callback(oct_hw->vqs[i].cb.private);
+ }
+ }
+
+ return IRQ_HANDLED;
+}
+
+static void octep_free_irqs(struct octep_hw *oct_hw)
+{
+ struct pci_dev *pdev = oct_hw->pdev;
+
+ if (oct_hw->irq != -1) {
+ devm_free_irq(&pdev->dev, oct_hw->irq, oct_hw);
+ oct_hw->irq = -1;
+ }
+ pci_free_irq_vectors(pdev);
+}
+
+static int octep_request_irqs(struct octep_hw *oct_hw)
+{
+ struct pci_dev *pdev = oct_hw->pdev;
+ int ret, irq;
+
+ /* Currently HW device provisions one IRQ per VF, hence
+ * allocate one IRQ for all virtqueues call interface.
+ */
+ ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSIX);
+ if (ret < 0) {
+ dev_err(&pdev->dev, "Failed to alloc msix vector");
+ return ret;
+ }
+
+ snprintf(oct_hw->vqs->msix_name, sizeof(oct_hw->vqs->msix_name),
+ OCTEP_VDPA_DRIVER_NAME "-vf-%d", pci_iov_vf_id(pdev));
+
+ irq = pci_irq_vector(pdev, 0);
+ ret = devm_request_irq(&pdev->dev, irq, octep_vdpa_intr_handler, 0,
+ oct_hw->vqs->msix_name, oct_hw);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to register interrupt handler\n");
+ goto free_irq_vec;
+ }
+ oct_hw->irq = irq;
+
+ return 0;
+
+free_irq_vec:
+ pci_free_irq_vectors(pdev);
+ return ret;
+}
+
+static u64 octep_vdpa_get_device_features(struct vdpa_device *vdpa_dev)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+
+ return oct_hw->features;
+}
+
+static int octep_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 features)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+ int ret;
+
+ pr_debug("Driver Features: %llx\n", features);
+
+ ret = octep_verify_features(features);
+ if (ret) {
+ dev_warn(&oct_hw->pdev->dev,
+ "Must negotiate minimum features 0x%llx for this device",
+ BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_NOTIFICATION_DATA) |
+ BIT_ULL(VIRTIO_F_RING_PACKED));
+ return ret;
+ }
+ octep_hw_set_drv_features(oct_hw, features);
+
+ return 0;
+}
+
+static u64 octep_vdpa_get_driver_features(struct vdpa_device *vdpa_dev)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+
+ return octep_hw_get_drv_features(oct_hw);
+}
+
+static u8 octep_vdpa_get_status(struct vdpa_device *vdpa_dev)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+
+ return octep_hw_get_status(oct_hw);
+}
+
+static void octep_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+ u8 status_old;
+
+ status_old = octep_hw_get_status(oct_hw);
+
+ if (status_old == status)
+ return;
+
+ if ((status & VIRTIO_CONFIG_S_DRIVER_OK) &&
+ !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) {
+ if (octep_request_irqs(oct_hw))
+ status = status_old | VIRTIO_CONFIG_S_FAILED;
+ }
+ octep_hw_set_status(oct_hw, status);
+}
+
+static int octep_vdpa_reset(struct vdpa_device *vdpa_dev)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+ u8 status = octep_hw_get_status(oct_hw);
+ u16 qid;
+
+ if (status == 0)
+ return 0;
+
+ for (qid = 0; qid < oct_hw->nr_vring; qid++) {
+ oct_hw->vqs[qid].cb.callback = NULL;
+ oct_hw->vqs[qid].cb.private = NULL;
+ oct_hw->config_cb.callback = NULL;
+ oct_hw->config_cb.private = NULL;
+ }
+ octep_hw_reset(oct_hw);
+
+ if (status & VIRTIO_CONFIG_S_DRIVER_OK)
+ octep_free_irqs(oct_hw);
+
+ return 0;
+}
+
+static u16 octep_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+
+ return octep_get_vq_size(oct_hw);
+}
+
+static int octep_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
+ struct vdpa_vq_state *state)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+
+ return octep_get_vq_state(oct_hw, qid, state);
+}
+
+static int octep_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
+ const struct vdpa_vq_state *state)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+
+ return octep_set_vq_state(oct_hw, qid, state);
+}
+
+static void octep_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid, struct vdpa_callback *cb)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+
+ oct_hw->vqs[qid].cb = *cb;
+}
+
+static void octep_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, u16 qid, bool ready)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+
+ octep_set_vq_ready(oct_hw, qid, ready);
+}
+
+static bool octep_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+
+ return octep_get_vq_ready(oct_hw, qid);
+}
+
+static void octep_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, u32 num)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+
+ octep_set_vq_num(oct_hw, qid, num);
+}
+
+static int octep_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid, u64 desc_area,
+ u64 driver_area, u64 device_area)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+
+ pr_debug("qid[%d]: desc_area: %llx\n", qid, desc_area);
+ pr_debug("qid[%d]: driver_area: %llx\n", qid, driver_area);
+ pr_debug("qid[%d]: device_area: %llx\n\n", qid, device_area);
+
+ return octep_set_vq_address(oct_hw, qid, desc_area, driver_area, device_area);
+}
+
+static void octep_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid)
+{
+ /* Not supported */
+}
+
+static void octep_vdpa_kick_vq_with_data(struct vdpa_device *vdpa_dev, u32 data)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+ u16 idx = data & 0xFFFF;
+
+ vp_iowrite32(data, oct_hw->vqs[idx].notify_addr);
+}
+
+static u32 octep_vdpa_get_generation(struct vdpa_device *vdpa_dev)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+
+ return vp_ioread8(&oct_hw->common_cfg->config_generation);
+}
+
+static u32 octep_vdpa_get_device_id(struct vdpa_device *vdpa_dev)
+{
+ return VIRTIO_ID_NET;
+}
+
+static u32 octep_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev)
+{
+ return PCI_VENDOR_ID_CAVIUM;
+}
+
+static u32 octep_vdpa_get_vq_align(struct vdpa_device *vdpa_dev)
+{
+ return PAGE_SIZE;
+}
+
+static size_t octep_vdpa_get_config_size(struct vdpa_device *vdpa_dev)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+
+ return oct_hw->config_size;
+}
+
+static void octep_vdpa_get_config(struct vdpa_device *vdpa_dev, unsigned int offset, void *buf,
+ unsigned int len)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+
+ octep_read_dev_config(oct_hw, offset, buf, len);
+}
+
+static void octep_vdpa_set_config(struct vdpa_device *vdpa_dev, unsigned int offset,
+ const void *buf, unsigned int len)
+{
+ /* Not supported */
+}
+
+static void octep_vdpa_set_config_cb(struct vdpa_device *vdpa_dev, struct vdpa_callback *cb)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+
+ oct_hw->config_cb.callback = cb->callback;
+ oct_hw->config_cb.private = cb->private;
+}
+
+static struct vdpa_notification_area octep_get_vq_notification(struct vdpa_device *vdpa_dev,
+ u16 idx)
+{
+ struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
+ struct vdpa_notification_area area;
+
+ area.addr = oct_hw->vqs[idx].notify_pa;
+ area.size = PAGE_SIZE;
+
+ return area;
+}
+
+static struct vdpa_config_ops octep_vdpa_ops = {
+ .get_device_features = octep_vdpa_get_device_features,
+ .set_driver_features = octep_vdpa_set_driver_features,
+ .get_driver_features = octep_vdpa_get_driver_features,
+ .get_status = octep_vdpa_get_status,
+ .set_status = octep_vdpa_set_status,
+ .reset = octep_vdpa_reset,
+ .get_vq_num_max = octep_vdpa_get_vq_num_max,
+ .get_vq_state = octep_vdpa_get_vq_state,
+ .set_vq_state = octep_vdpa_set_vq_state,
+ .set_vq_cb = octep_vdpa_set_vq_cb,
+ .set_vq_ready = octep_vdpa_set_vq_ready,
+ .get_vq_ready = octep_vdpa_get_vq_ready,
+ .set_vq_num = octep_vdpa_set_vq_num,
+ .set_vq_address = octep_vdpa_set_vq_address,
+ .get_vq_irq = NULL,
+ .kick_vq = octep_vdpa_kick_vq,
+ .kick_vq_with_data = octep_vdpa_kick_vq_with_data,
+ .get_generation = octep_vdpa_get_generation,
+ .get_device_id = octep_vdpa_get_device_id,
+ .get_vendor_id = octep_vdpa_get_vendor_id,
+ .get_vq_align = octep_vdpa_get_vq_align,
+ .get_config_size = octep_vdpa_get_config_size,
+ .get_config = octep_vdpa_get_config,
+ .set_config = octep_vdpa_set_config,
+ .set_config_cb = octep_vdpa_set_config_cb,
+ .get_vq_notification = octep_get_vq_notification,
+};
+
+static int octep_iomap_region(struct pci_dev *pdev, u8 __iomem **tbl, u8 bar)
+{
+ int ret;
+
+ ret = pci_request_region(pdev, bar, OCTEP_VDPA_DRIVER_NAME);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to request BAR:%u region\n", bar);
+ return ret;
+ }
+
+ tbl[bar] = pci_iomap(pdev, bar, pci_resource_len(pdev, bar));
+ if (!tbl[bar]) {
+ dev_err(&pdev->dev, "Failed to iomap BAR:%u\n", bar);
+ pci_release_region(pdev, bar);
+ ret = -ENOMEM;
+ }
+
+ return ret;
+}
+
+static void octep_iounmap_region(struct pci_dev *pdev, u8 __iomem **tbl, u8 bar)
+{
+ pci_iounmap(pdev, tbl[bar]);
+ pci_release_region(pdev, bar);
+}
+
+static void octep_vdpa_pf_bar_shrink(struct octep_pf *octpf)
+{
+ struct pci_dev *pf_dev = octpf->pdev;
+ struct resource *res = pf_dev->resource + PCI_STD_RESOURCES + 4;
+ struct pci_bus_region bus_region;
+
+ octpf->res.start = res->start;
+ octpf->res.end = res->end;
+ octpf->vf_base = res->start;
+
+ bus_region.start = res->start;
+ bus_region.end = res->start - 1;
+
+ pcibios_bus_to_resource(pf_dev->bus, res, &bus_region);
+}
+
+static void octep_vdpa_pf_bar_expand(struct octep_pf *octpf)
+{
+ struct pci_dev *pf_dev = octpf->pdev;
+ struct resource *res = pf_dev->resource + PCI_STD_RESOURCES + 4;
+ struct pci_bus_region bus_region;
+
+ bus_region.start = octpf->res.start;
+ bus_region.end = octpf->res.end;
+
+ pcibios_bus_to_resource(pf_dev->bus, res, &bus_region);
+}
+
+static void octep_vdpa_remove_pf(struct pci_dev *pdev)
+{
+ struct octep_pf *octpf = pci_get_drvdata(pdev);
+
+ pci_disable_sriov(pdev);
+
+ if (octpf->base[OCTEP_HW_CAPS_BAR])
+ octep_iounmap_region(pdev, octpf->base, OCTEP_HW_CAPS_BAR);
+
+ if (octpf->base[OCTEP_HW_MBOX_BAR])
+ octep_iounmap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR);
+
+ octep_vdpa_pf_bar_expand(octpf);
+}
+
+static void octep_vdpa_vf_bar_shrink(struct pci_dev *pdev)
+{
+ struct resource *vf_res = pdev->resource + PCI_STD_RESOURCES + 4;
+
+ memset(vf_res, 0, sizeof(*vf_res));
+}
+
+static void octep_vdpa_remove_vf(struct pci_dev *pdev)
+{
+ struct octep_vdpa_mgmt_dev *mgmt_dev = pci_get_drvdata(pdev);
+ struct octep_hw *oct_hw;
+ int status;
+
+ oct_hw = &mgmt_dev->oct_hw;
+ status = atomic_read(&mgmt_dev->status);
+ atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_UNINIT);
+
+ cancel_work_sync(&mgmt_dev->setup_task);
+ if (status == OCTEP_VDPA_DEV_STATUS_READY)
+ vdpa_mgmtdev_unregister(&mgmt_dev->mdev);
+
+ if (oct_hw->base[OCTEP_HW_CAPS_BAR])
+ octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR);
+
+ if (oct_hw->base[OCTEP_HW_MBOX_BAR])
+ octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_MBOX_BAR);
+
+ octep_vdpa_vf_bar_shrink(pdev);
+}
+
+static void octep_vdpa_remove(struct pci_dev *pdev)
+{
+ if (pdev->is_virtfn)
+ octep_vdpa_remove_vf(pdev);
+ else
+ octep_vdpa_remove_pf(pdev);
+}
+
+static int octep_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
+ const struct vdpa_dev_set_config *config)
+{
+ struct octep_vdpa_mgmt_dev *mgmt_dev = container_of(mdev, struct octep_vdpa_mgmt_dev, mdev);
+ struct octep_hw *oct_hw = &mgmt_dev->oct_hw;
+ struct pci_dev *pdev = oct_hw->pdev;
+ struct vdpa_device *vdpa_dev;
+ struct octep_vdpa *oct_vdpa;
+ u64 device_features;
+ int ret;
+
+ oct_vdpa = vdpa_alloc_device(struct octep_vdpa, vdpa, &pdev->dev, &octep_vdpa_ops, 1, 1,
+ NULL, false);
+ if (IS_ERR(oct_vdpa)) {
+ dev_err(&pdev->dev, "Failed to allocate vDPA structure for octep vdpa device");
+ return PTR_ERR(oct_vdpa);
+ }
+
+ oct_vdpa->pdev = pdev;
+ oct_vdpa->vdpa.dma_dev = &pdev->dev;
+ oct_vdpa->vdpa.mdev = mdev;
+ oct_vdpa->oct_hw = oct_hw;
+ vdpa_dev = &oct_vdpa->vdpa;
+
+ device_features = oct_hw->features;
+ if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
+ if (config->device_features & ~device_features) {
+ dev_err(&pdev->dev, "The provisioned features 0x%llx are not supported by this device with features 0x%llx\n",
+ config->device_features, device_features);
+ ret = -EINVAL;
+ goto vdpa_dev_put;
+ }
+ device_features &= config->device_features;
+ }
+
+ oct_hw->features = device_features;
+ dev_info(&pdev->dev, "Vdpa management device features : %llx\n", device_features);
+
+ ret = octep_verify_features(device_features);
+ if (ret) {
+ dev_warn(mdev->device,
+ "Must provision minimum features 0x%llx for this device",
+ BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM) |
+ BIT_ULL(VIRTIO_F_NOTIFICATION_DATA) | BIT_ULL(VIRTIO_F_RING_PACKED));
+ goto vdpa_dev_put;
+ }
+ if (name)
+ ret = dev_set_name(&vdpa_dev->dev, "%s", name);
+ else
+ ret = dev_set_name(&vdpa_dev->dev, "vdpa%u", vdpa_dev->index);
+
+ ret = _vdpa_register_device(&oct_vdpa->vdpa, oct_hw->nr_vring);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to register to vDPA bus");
+ goto vdpa_dev_put;
+ }
+ return 0;
+
+vdpa_dev_put:
+ put_device(&oct_vdpa->vdpa.dev);
+ return ret;
+}
+
+static void octep_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *vdpa_dev)
+{
+ _vdpa_unregister_device(vdpa_dev);
+}
+
+static const struct vdpa_mgmtdev_ops octep_vdpa_mgmt_dev_ops = {
+ .dev_add = octep_vdpa_dev_add,
+ .dev_del = octep_vdpa_dev_del
+};
+
+static bool get_device_ready_status(u8 __iomem *addr)
+{
+ u64 signature = readq(addr + OCTEP_VF_MBOX_DATA(0));
+
+ if (signature == OCTEP_DEV_READY_SIGNATURE) {
+ writeq(0, addr + OCTEP_VF_MBOX_DATA(0));
+ return true;
+ }
+
+ return false;
+}
+
+static struct virtio_device_id id_table[] = {
+ { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
+ { 0 },
+};
+
+static void octep_vdpa_setup_task(struct work_struct *work)
+{
+ struct octep_vdpa_mgmt_dev *mgmt_dev = container_of(work, struct octep_vdpa_mgmt_dev,
+ setup_task);
+ struct pci_dev *pdev = mgmt_dev->pdev;
+ struct device *dev = &pdev->dev;
+ struct octep_hw *oct_hw;
+ unsigned long timeout;
+ int ret;
+
+ oct_hw = &mgmt_dev->oct_hw;
+
+ atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_WAIT_FOR_BAR_INIT);
+
+ /* Wait for a maximum of 5 sec */
+ timeout = jiffies + msecs_to_jiffies(5000);
+ while (!time_after(jiffies, timeout)) {
+ if (get_device_ready_status(oct_hw->base[OCTEP_HW_MBOX_BAR])) {
+ atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_INIT);
+ break;
+ }
+
+ if (atomic_read(&mgmt_dev->status) >= OCTEP_VDPA_DEV_STATUS_READY) {
+ dev_info(dev, "Stopping vDPA setup task.\n");
+ return;
+ }
+
+ usleep_range(1000, 1500);
+ }
+
+ if (atomic_read(&mgmt_dev->status) != OCTEP_VDPA_DEV_STATUS_INIT) {
+ dev_err(dev, "BAR initialization is timed out\n");
+ return;
+ }
+
+ ret = octep_iomap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR);
+ if (ret)
+ return;
+
+ ret = octep_hw_caps_read(oct_hw, pdev);
+ if (ret < 0)
+ goto unmap_region;
+
+ mgmt_dev->mdev.ops = &octep_vdpa_mgmt_dev_ops;
+ mgmt_dev->mdev.id_table = id_table;
+ mgmt_dev->mdev.max_supported_vqs = oct_hw->nr_vring;
+ mgmt_dev->mdev.supported_features = oct_hw->features;
+ mgmt_dev->mdev.config_attr_mask = (1 << VDPA_ATTR_DEV_FEATURES);
+ mgmt_dev->mdev.device = dev;
+
+ ret = vdpa_mgmtdev_register(&mgmt_dev->mdev);
+ if (ret) {
+ dev_err(dev, "Failed to register vdpa management interface\n");
+ goto unmap_region;
+ }
+
+ atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_READY);
+
+ return;
+
+unmap_region:
+ octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR);
+ oct_hw->base[OCTEP_HW_CAPS_BAR] = NULL;
+}
+
+static int octep_vdpa_probe_vf(struct pci_dev *pdev)
+{
+ struct octep_vdpa_mgmt_dev *mgmt_dev;
+ struct device *dev = &pdev->dev;
+ int ret;
+
+ ret = pcim_enable_device(pdev);
+ if (ret) {
+ dev_err(dev, "Failed to enable device\n");
+ return ret;
+ }
+
+ ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
+ if (ret) {
+ dev_err(dev, "No usable DMA configuration\n");
+ return ret;
+ }
+ pci_set_master(pdev);
+
+ mgmt_dev = devm_kzalloc(dev, sizeof(struct octep_vdpa_mgmt_dev), GFP_KERNEL);
+ if (!mgmt_dev)
+ return -ENOMEM;
+
+ ret = octep_iomap_region(pdev, mgmt_dev->oct_hw.base, OCTEP_HW_MBOX_BAR);
+ if (ret)
+ return ret;
+
+ mgmt_dev->pdev = pdev;
+ pci_set_drvdata(pdev, mgmt_dev);
+
+ atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_ALLOC);
+ INIT_WORK(&mgmt_dev->setup_task, octep_vdpa_setup_task);
+ schedule_work(&mgmt_dev->setup_task);
+ dev_info(&pdev->dev, "octep vdpa mgmt device setup task is queued\n");
+
+ return 0;
+}
+
+static void octep_vdpa_assign_barspace(struct pci_dev *vf_dev, struct pci_dev *pf_dev, u8 idx)
+{
+ struct resource *vf_res = vf_dev->resource + PCI_STD_RESOURCES + 4;
+ struct resource *pf_res = pf_dev->resource + PCI_STD_RESOURCES + 4;
+ struct octep_pf *pf = pci_get_drvdata(pf_dev);
+ struct pci_bus_region bus_region;
+
+ vf_res->name = pci_name(vf_dev);
+ vf_res->flags = pf_res->flags;
+ vf_res->parent = (pf_dev->resource + PCI_STD_RESOURCES)->parent;
+
+ bus_region.start = pf->vf_base + idx * pf->vf_stride;
+ bus_region.end = bus_region.start + pf->vf_stride - 1;
+ pcibios_bus_to_resource(vf_dev->bus, vf_res, &bus_region);
+}
+
+static int octep_sriov_enable(struct pci_dev *pdev, int num_vfs)
+{
+ struct octep_pf *pf = pci_get_drvdata(pdev);
+ u8 __iomem *addr = pf->base[OCTEP_HW_MBOX_BAR];
+ struct pci_dev *vf_pdev = NULL;
+ bool done = false;
+ int index = 0;
+ int ret, i;
+
+ ret = pci_enable_sriov(pdev, num_vfs);
+ if (ret)
+ return ret;
+
+ pf->enabled_vfs = num_vfs;
+
+ while ((vf_pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, PCI_ANY_ID, vf_pdev))) {
+ if (vf_pdev->device != pf->vf_devid)
+ continue;
+
+ octep_vdpa_assign_barspace(vf_pdev, pdev, index);
+ if (++index == num_vfs) {
+ done = true;
+ break;
+ }
+ }
+
+ if (done) {
+ for (i = 0; i < pf->enabled_vfs; i++)
+ writeq(OCTEP_DEV_READY_SIGNATURE, addr + OCTEP_PF_MBOX_DATA(i));
+ }
+
+ return num_vfs;
+}
+
+static int octep_sriov_disable(struct pci_dev *pdev)
+{
+ struct octep_pf *pf = pci_get_drvdata(pdev);
+
+ if (!pci_num_vf(pdev))
+ return 0;
+
+ pci_disable_sriov(pdev);
+ pf->enabled_vfs = 0;
+
+ return 0;
+}
+
+static int octep_vdpa_sriov_configure(struct pci_dev *pdev, int num_vfs)
+{
+ if (num_vfs > 0)
+ return octep_sriov_enable(pdev, num_vfs);
+ else
+ return octep_sriov_disable(pdev);
+}
+
+static u16 octep_get_vf_devid(struct pci_dev *pdev)
+{
+ u16 did;
+
+ switch (pdev->device) {
+ case OCTEP_VDPA_DEVID_CN106K_PF:
+ did = OCTEP_VDPA_DEVID_CN106K_VF;
+ break;
+ case OCTEP_VDPA_DEVID_CN105K_PF:
+ did = OCTEP_VDPA_DEVID_CN105K_VF;
+ break;
+ case OCTEP_VDPA_DEVID_CN103K_PF:
+ did = OCTEP_VDPA_DEVID_CN103K_VF;
+ break;
+ default:
+ did = 0xFFFF;
+ break;
+ }
+
+ return did;
+}
+
+static int octep_vdpa_pf_setup(struct octep_pf *octpf)
+{
+ u8 __iomem *addr = octpf->base[OCTEP_HW_MBOX_BAR];
+ struct pci_dev *pdev = octpf->pdev;
+ int totalvfs;
+ size_t len;
+ u64 val;
+
+ totalvfs = pci_sriov_get_totalvfs(pdev);
+ if (unlikely(!totalvfs)) {
+ dev_info(&pdev->dev, "Total VFs are %d in PF sriov configuration\n", totalvfs);
+ return 0;
+ }
+
+ addr = octpf->base[OCTEP_HW_MBOX_BAR];
+ val = readq(addr + OCTEP_EPF_RINFO(0));
+ if (val == 0) {
+ dev_err(&pdev->dev, "Invalid device configuration\n");
+ return -EINVAL;
+ }
+
+ if (OCTEP_EPF_RINFO_RPVF(val) != BIT_ULL(0)) {
+ val &= ~GENMASK_ULL(35, 32);
+ val |= BIT_ULL(32);
+ writeq(val, addr + OCTEP_EPF_RINFO(0));
+ }
+
+ len = pci_resource_len(pdev, OCTEP_HW_CAPS_BAR);
+
+ octpf->vf_stride = len / totalvfs;
+ octpf->vf_devid = octep_get_vf_devid(pdev);
+
+ octep_vdpa_pf_bar_shrink(octpf);
+
+ return 0;
+}
+
+static int octep_vdpa_probe_pf(struct pci_dev *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct octep_pf *octpf;
+ int ret;
+
+ ret = pcim_enable_device(pdev);
+ if (ret) {
+ dev_err(dev, "Failed to enable device\n");
+ return ret;
+ }
+
+ ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
+ if (ret) {
+ dev_err(dev, "No usable DMA configuration\n");
+ return ret;
+ }
+ octpf = devm_kzalloc(dev, sizeof(*octpf), GFP_KERNEL);
+ if (!octpf)
+ return -ENOMEM;
+
+ ret = octep_iomap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR);
+ if (ret)
+ return ret;
+
+ pci_set_master(pdev);
+ pci_set_drvdata(pdev, octpf);
+ octpf->pdev = pdev;
+
+ ret = octep_vdpa_pf_setup(octpf);
+ if (ret)
+ goto unmap_region;
+
+ return 0;
+
+unmap_region:
+ octep_iounmap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR);
+ return ret;
+}
+
+static int octep_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+ if (pdev->is_virtfn)
+ return octep_vdpa_probe_vf(pdev);
+ else
+ return octep_vdpa_probe_pf(pdev);
+}
+
+static struct pci_device_id octep_pci_vdpa_map[] = {
+ { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN106K_PF) },
+ { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN106K_VF) },
+ { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN105K_PF) },
+ { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN105K_VF) },
+ { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN103K_PF) },
+ { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN103K_VF) },
+ { 0 },
+};
+
+static struct pci_driver octep_pci_vdpa = {
+ .name = OCTEP_VDPA_DRIVER_NAME,
+ .id_table = octep_pci_vdpa_map,
+ .probe = octep_vdpa_probe,
+ .remove = octep_vdpa_remove,
+ .sriov_configure = octep_vdpa_sriov_configure
+};
+
+module_pci_driver(octep_pci_vdpa);
+
+MODULE_AUTHOR("Marvell");
+MODULE_DESCRIPTION("Marvell Octeon PCIe endpoint vDPA driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c
index 8d391947eb8d..1ca445e31acb 100644
--- a/drivers/vdpa/vdpa.c
+++ b/drivers/vdpa/vdpa.c
@@ -1538,4 +1538,5 @@ core_initcall(vdpa_init);
module_exit(vdpa_exit);
MODULE_AUTHOR("Jason Wang <jasowang@redhat.com>");
+MODULE_DESCRIPTION("vDPA bus");
MODULE_LICENSE("GPL v2");
generated by cgit v1.2.3 (git 2.39.0) at 2025-02-14 03:29:57 +0300