diff options
Diffstat (limited to 'drivers/infiniband')
71 files changed, 2892 insertions, 1511 deletions
diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c index f4e8185bccd3..a5b4cf030c11 100644 --- a/drivers/infiniband/core/addr.c +++ b/drivers/infiniband/core/addr.c @@ -243,8 +243,7 @@ void rdma_copy_addr(struct rdma_dev_addr *dev_addr, EXPORT_SYMBOL(rdma_copy_addr); int rdma_translate_ip(const struct sockaddr *addr, - struct rdma_dev_addr *dev_addr, - u16 *vlan_id) + struct rdma_dev_addr *dev_addr) { struct net_device *dev; @@ -266,9 +265,6 @@ int rdma_translate_ip(const struct sockaddr *addr, return -EADDRNOTAVAIL; rdma_copy_addr(dev_addr, dev, NULL); - dev_addr->bound_dev_if = dev->ifindex; - if (vlan_id) - *vlan_id = rdma_vlan_dev_vlan_id(dev); dev_put(dev); break; #if IS_ENABLED(CONFIG_IPV6) @@ -279,9 +275,6 @@ int rdma_translate_ip(const struct sockaddr *addr, &((const struct sockaddr_in6 *)addr)->sin6_addr, dev, 1)) { rdma_copy_addr(dev_addr, dev, NULL); - dev_addr->bound_dev_if = dev->ifindex; - if (vlan_id) - *vlan_id = rdma_vlan_dev_vlan_id(dev); break; } } @@ -481,7 +474,7 @@ static int addr_resolve_neigh(struct dst_entry *dst, if (dst->dev->flags & IFF_LOOPBACK) { int ret; - ret = rdma_translate_ip(dst_in, addr, NULL); + ret = rdma_translate_ip(dst_in, addr); if (!ret) memcpy(addr->dst_dev_addr, addr->src_dev_addr, MAX_ADDR_LEN); @@ -558,7 +551,7 @@ static int addr_resolve(struct sockaddr *src_in, } if (ndev->flags & IFF_LOOPBACK) { - ret = rdma_translate_ip(dst_in, addr, NULL); + ret = rdma_translate_ip(dst_in, addr); /* * Put the loopback device and get the translated * device instead. @@ -744,7 +737,6 @@ void rdma_addr_cancel(struct rdma_dev_addr *addr) EXPORT_SYMBOL(rdma_addr_cancel); struct resolve_cb_context { - struct rdma_dev_addr *addr; struct completion comp; int status; }; @@ -752,39 +744,31 @@ struct resolve_cb_context { static void resolve_cb(int status, struct sockaddr *src_addr, struct rdma_dev_addr *addr, void *context) { - if (!status) - memcpy(((struct resolve_cb_context *)context)->addr, - addr, sizeof(struct rdma_dev_addr)); ((struct resolve_cb_context *)context)->status = status; complete(&((struct resolve_cb_context *)context)->comp); } int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid, const union ib_gid *dgid, - u8 *dmac, u16 *vlan_id, int *if_index, + u8 *dmac, const struct net_device *ndev, int *hoplimit) { - int ret = 0; struct rdma_dev_addr dev_addr; struct resolve_cb_context ctx; - struct net_device *dev; - union { struct sockaddr _sockaddr; struct sockaddr_in _sockaddr_in; struct sockaddr_in6 _sockaddr_in6; } sgid_addr, dgid_addr; - + int ret; rdma_gid2ip(&sgid_addr._sockaddr, sgid); rdma_gid2ip(&dgid_addr._sockaddr, dgid); memset(&dev_addr, 0, sizeof(dev_addr)); - if (if_index) - dev_addr.bound_dev_if = *if_index; + dev_addr.bound_dev_if = ndev->ifindex; dev_addr.net = &init_net; - ctx.addr = &dev_addr; init_completion(&ctx.comp); ret = rdma_resolve_ip(&self, &sgid_addr._sockaddr, &dgid_addr._sockaddr, &dev_addr, 1000, resolve_cb, &ctx); @@ -798,42 +782,9 @@ int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid, return ret; memcpy(dmac, dev_addr.dst_dev_addr, ETH_ALEN); - dev = dev_get_by_index(&init_net, dev_addr.bound_dev_if); - if (!dev) - return -ENODEV; - if (if_index) - *if_index = dev_addr.bound_dev_if; - if (vlan_id) - *vlan_id = rdma_vlan_dev_vlan_id(dev); - if (hoplimit) - *hoplimit = dev_addr.hoplimit; - dev_put(dev); - return ret; -} -EXPORT_SYMBOL(rdma_addr_find_l2_eth_by_grh); - -int rdma_addr_find_smac_by_sgid(union ib_gid *sgid, u8 *smac, u16 *vlan_id) -{ - int ret = 0; - struct rdma_dev_addr dev_addr; - union { - struct sockaddr _sockaddr; - struct sockaddr_in _sockaddr_in; - struct sockaddr_in6 _sockaddr_in6; - } gid_addr; - - rdma_gid2ip(&gid_addr._sockaddr, sgid); - - memset(&dev_addr, 0, sizeof(dev_addr)); - dev_addr.net = &init_net; - ret = rdma_translate_ip(&gid_addr._sockaddr, &dev_addr, vlan_id); - if (ret) - return ret; - - memcpy(smac, dev_addr.src_dev_addr, ETH_ALEN); - return ret; + *hoplimit = dev_addr.hoplimit; + return 0; } -EXPORT_SYMBOL(rdma_addr_find_smac_by_sgid); static int netevent_callback(struct notifier_block *self, unsigned long event, void *ctx) diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c index 77515638c55c..7babdbceb6d0 100644 --- a/drivers/infiniband/core/cache.c +++ b/drivers/infiniband/core/cache.c @@ -573,27 +573,24 @@ static int ib_cache_gid_find_by_filter(struct ib_device *ib_dev, struct ib_gid_attr attr; if (table->data_vec[i].props & GID_TABLE_ENTRY_INVALID) - goto next; + continue; if (memcmp(gid, &table->data_vec[i].gid, sizeof(*gid))) - goto next; + continue; memcpy(&attr, &table->data_vec[i].attr, sizeof(attr)); - if (filter(gid, &attr, context)) + if (filter(gid, &attr, context)) { found = true; - -next: - if (found) + if (index) + *index = i; break; + } } read_unlock_irqrestore(&table->rwlock, flags); if (!found) return -ENOENT; - - if (index) - *index = i; return 0; } @@ -883,7 +880,6 @@ int ib_find_gid_by_filter(struct ib_device *device, port_num, filter, context, index); } -EXPORT_SYMBOL(ib_find_gid_by_filter); int ib_get_cached_pkey(struct ib_device *device, u8 port_num, diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index f6b159d79977..b8f8d3128a53 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -452,13 +452,14 @@ static void cm_set_private_data(struct cm_id_private *cm_id_priv, cm_id_priv->private_data_len = private_data_len; } -static void cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc, - struct ib_grh *grh, struct cm_av *av) +static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc, + struct ib_grh *grh, struct cm_av *av) { av->port = port; av->pkey_index = wc->pkey_index; - ib_init_ah_from_wc(port->cm_dev->ib_device, port->port_num, wc, - grh, &av->ah_attr); + return ib_init_ah_attr_from_wc(port->cm_dev->ib_device, + port->port_num, wc, + grh, &av->ah_attr); } static int cm_init_av_by_path(struct sa_path_rec *path, struct cm_av *av, @@ -494,8 +495,11 @@ static int cm_init_av_by_path(struct sa_path_rec *path, struct cm_av *av, return ret; av->port = port; - ib_init_ah_from_path(cm_dev->ib_device, port->port_num, path, - &av->ah_attr); + ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path, + &av->ah_attr); + if (ret) + return ret; + av->timeout = path->packet_life_time + 1; spin_lock_irqsave(&cm.lock, flags); @@ -1560,6 +1564,37 @@ static u16 cm_get_bth_pkey(struct cm_work *work) return pkey; } +/** + * Convert OPA SGID to IB SGID + * ULPs (such as IPoIB) do not understand OPA GIDs and will + * reject them as the local_gid will not match the sgid. Therefore, + * change the pathrec's SGID to an IB SGID. + * + * @work: Work completion + * @path: Path record + */ +static void cm_opa_to_ib_sgid(struct cm_work *work, + struct sa_path_rec *path) +{ + struct ib_device *dev = work->port->cm_dev->ib_device; + struct ib_gid_attr gid_attr; + u8 port_num = work->port->port_num; + + if (rdma_cap_opa_ah(dev, port_num) && + (ib_is_opa_gid(&path->sgid))) { + union ib_gid sgid; + + if (ib_get_cached_gid(dev, port_num, 0, + &sgid, &gid_attr)) { + dev_warn(&dev->dev, + "Error updating sgid in CM request\n"); + return; + } + + path->sgid = sgid; + } +} + static void cm_format_req_event(struct cm_work *work, struct cm_id_private *cm_id_priv, struct ib_cm_id *listen_id) @@ -1573,10 +1608,13 @@ static void cm_format_req_event(struct cm_work *work, param->bth_pkey = cm_get_bth_pkey(work); param->port = cm_id_priv->av.port->port_num; param->primary_path = &work->path[0]; - if (cm_req_has_alt_path(req_msg)) + cm_opa_to_ib_sgid(work, param->primary_path); + if (cm_req_has_alt_path(req_msg)) { param->alternate_path = &work->path[1]; - else + cm_opa_to_ib_sgid(work, param->alternate_path); + } else { param->alternate_path = NULL; + } param->remote_ca_guid = req_msg->local_ca_guid; param->remote_qkey = be32_to_cpu(req_msg->local_qkey); param->remote_qpn = be32_to_cpu(cm_req_get_local_qpn(req_msg)); @@ -1826,9 +1864,11 @@ static int cm_req_handler(struct cm_work *work) cm_id_priv = container_of(cm_id, struct cm_id_private, id); cm_id_priv->id.remote_id = req_msg->local_comm_id; - cm_init_av_for_response(work->port, work->mad_recv_wc->wc, - work->mad_recv_wc->recv_buf.grh, - &cm_id_priv->av); + ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc, + work->mad_recv_wc->recv_buf.grh, + &cm_id_priv->av); + if (ret) + goto destroy; cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv-> id.local_id); if (IS_ERR(cm_id_priv->timewait_info)) { @@ -1841,9 +1881,10 @@ static int cm_req_handler(struct cm_work *work) listen_cm_id_priv = cm_match_req(work, cm_id_priv); if (!listen_cm_id_priv) { + pr_debug("%s: local_id %d, no listen_cm_id_priv\n", __func__, + be32_to_cpu(cm_id->local_id)); ret = -EINVAL; - kfree(cm_id_priv->timewait_info); - goto destroy; + goto free_timeinfo; } cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler; @@ -1861,56 +1902,52 @@ static int cm_req_handler(struct cm_work *work) work->port->port_num, grh->sgid_index, &gid, &gid_attr); - if (!ret) { - if (gid_attr.ndev) { - work->path[0].rec_type = - sa_conv_gid_to_pathrec_type(gid_attr.gid_type); - sa_path_set_ifindex(&work->path[0], - gid_attr.ndev->ifindex); - sa_path_set_ndev(&work->path[0], - dev_net(gid_attr.ndev)); + if (ret) { + if (gid_attr.ndev) dev_put(gid_attr.ndev); - } else { - cm_path_set_rec_type(work->port->cm_dev->ib_device, - work->port->port_num, - &work->path[0], - &req_msg->primary_local_gid); - } - if (cm_req_has_alt_path(req_msg)) - work->path[1].rec_type = work->path[0].rec_type; - cm_format_paths_from_req(req_msg, &work->path[0], - &work->path[1]); - if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE) - sa_path_set_dmac(&work->path[0], - cm_id_priv->av.ah_attr.roce.dmac); - work->path[0].hop_limit = grh->hop_limit; - ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av, - cm_id_priv); + ib_send_cm_rej(cm_id, IB_CM_REJ_UNSUPPORTED, NULL, 0, NULL, 0); + goto rejected; } + + if (gid_attr.ndev) { + work->path[0].rec_type = + sa_conv_gid_to_pathrec_type(gid_attr.gid_type); + sa_path_set_ifindex(&work->path[0], + gid_attr.ndev->ifindex); + sa_path_set_ndev(&work->path[0], + dev_net(gid_attr.ndev)); + dev_put(gid_attr.ndev); + } else { + cm_path_set_rec_type(work->port->cm_dev->ib_device, + work->port->port_num, + &work->path[0], + &req_msg->primary_local_gid); + } + if (cm_req_has_alt_path(req_msg)) + work->path[1].rec_type = work->path[0].rec_type; + cm_format_paths_from_req(req_msg, &work->path[0], + &work->path[1]); + if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE) + sa_path_set_dmac(&work->path[0], + cm_id_priv->av.ah_attr.roce.dmac); + work->path[0].hop_limit = grh->hop_limit; + ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av, + cm_id_priv); if (ret) { - int err = ib_get_cached_gid(work->port->cm_dev->ib_device, - work->port->port_num, 0, - &work->path[0].sgid, - &gid_attr); - if (!err && gid_attr.ndev) { - work->path[0].rec_type = - sa_conv_gid_to_pathrec_type(gid_attr.gid_type); - sa_path_set_ifindex(&work->path[0], - gid_attr.ndev->ifindex); - sa_path_set_ndev(&work->path[0], - dev_net(gid_attr.ndev)); - dev_put(gid_attr.ndev); - } else { - cm_path_set_rec_type(work->port->cm_dev->ib_device, - work->port->port_num, - &work->path[0], - &req_msg->primary_local_gid); - } - if (cm_req_has_alt_path(req_msg)) - work->path[1].rec_type = work->path[0].rec_type; - ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID, - &work->path[0].sgid, sizeof work->path[0].sgid, - NULL, 0); + int err; + + err = ib_get_cached_gid(work->port->cm_dev->ib_device, + work->port->port_num, 0, + &work->path[0].sgid, + NULL); + if (err) + ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID, + NULL, 0, NULL, 0); + else + ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID, + &work->path[0].sgid, + sizeof(work->path[0].sgid), + NULL, 0); goto rejected; } if (cm_req_has_alt_path(req_msg)) { @@ -1919,7 +1956,7 @@ static int cm_req_handler(struct cm_work *work) if (ret) { ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_ALT_GID, &work->path[0].sgid, - sizeof work->path[0].sgid, NULL, 0); + sizeof(work->path[0].sgid), NULL, 0); goto rejected; } } @@ -1945,6 +1982,8 @@ static int cm_req_handler(struct cm_work *work) rejected: atomic_dec(&cm_id_priv->refcount); cm_deref_id(listen_cm_id_priv); +free_timeinfo: + kfree(cm_id_priv->timewait_info); destroy: ib_destroy_cm_id(cm_id); return ret; @@ -1997,6 +2036,8 @@ int ib_send_cm_rep(struct ib_cm_id *cm_id, spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id->state != IB_CM_REQ_RCVD && cm_id->state != IB_CM_MRA_REQ_SENT) { + pr_debug("%s: local_comm_id %d, cm_id->state: %d\n", __func__, + be32_to_cpu(cm_id_priv->id.local_id), cm_id->state); ret = -EINVAL; goto out; } @@ -2063,6 +2104,8 @@ int ib_send_cm_rtu(struct ib_cm_id *cm_id, spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id->state != IB_CM_REP_RCVD && cm_id->state != IB_CM_MRA_REP_SENT) { + pr_debug("%s: local_id %d, cm_id->state %d\n", __func__, + be32_to_cpu(cm_id->local_id), cm_id->state); ret = -EINVAL; goto error; } @@ -2170,6 +2213,8 @@ static int cm_rep_handler(struct cm_work *work) cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id, 0); if (!cm_id_priv) { cm_dup_rep_handler(work); + pr_debug("%s: remote_comm_id %d, no cm_id_priv\n", __func__, + be32_to_cpu(rep_msg->remote_comm_id)); return -EINVAL; } @@ -2183,6 +2228,10 @@ static int cm_rep_handler(struct cm_work *work) default: spin_unlock_irq(&cm_id_priv->lock); ret = -EINVAL; + pr_debug("%s: cm_id_priv->id.state: %d, local_comm_id %d, remote_comm_id %d\n", + __func__, cm_id_priv->id.state, + be32_to_cpu(rep_msg->local_comm_id), + be32_to_cpu(rep_msg->remote_comm_id)); goto error; } @@ -2196,6 +2245,8 @@ static int cm_rep_handler(struct cm_work *work) spin_unlock(&cm.lock); spin_unlock_irq(&cm_id_priv->lock); ret = -EINVAL; + pr_debug("%s: Failed to insert remote id %d\n", __func__, + be32_to_cpu(rep_msg->remote_comm_id)); goto error; } /* Check for a stale connection. */ @@ -2213,6 +2264,10 @@ static int cm_rep_handler(struct cm_work *work) IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP, NULL, 0); ret = -EINVAL; + pr_debug("%s: Stale connection. local_comm_id %d, remote_comm_id %d\n", + __func__, be32_to_cpu(rep_msg->local_comm_id), + be32_to_cpu(rep_msg->remote_comm_id)); + if (cur_cm_id_priv) { cm_id = &cur_cm_id_priv->id; ib_send_cm_dreq(cm_id, NULL, 0); @@ -2359,6 +2414,8 @@ int ib_send_cm_dreq(struct ib_cm_id *cm_id, cm_id_priv = container_of(cm_id, struct cm_id_private, id); spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id->state != IB_CM_ESTABLISHED) { + pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__, + be32_to_cpu(cm_id->local_id), cm_id->state); ret = -EINVAL; goto out; } @@ -2428,6 +2485,8 @@ int ib_send_cm_drep(struct ib_cm_id *cm_id, if (cm_id->state != IB_CM_DREQ_RCVD) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); kfree(data); + pr_debug("%s: local_id %d, cm_idcm_id->state(%d) != IB_CM_DREQ_RCVD\n", + __func__, be32_to_cpu(cm_id->local_id), cm_id->state); return -EINVAL; } @@ -2493,6 +2552,9 @@ static int cm_dreq_handler(struct cm_work *work) atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. counter[CM_DREQ_COUNTER]); cm_issue_drep(work->port, work->mad_recv_wc); + pr_debug("%s: no cm_id_priv, local_comm_id %d, remote_comm_id %d\n", + __func__, be32_to_cpu(dreq_msg->local_comm_id), + be32_to_cpu(dreq_msg->remote_comm_id)); return -EINVAL; } @@ -2535,6 +2597,9 @@ static int cm_dreq_handler(struct cm_work *work) counter[CM_DREQ_COUNTER]); goto unlock; default: + pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", + __func__, be32_to_cpu(cm_id_priv->id.local_id), + cm_id_priv->id.state); goto unlock; } cm_id_priv->id.state = IB_CM_DREQ_RCVD; @@ -2638,6 +2703,8 @@ int ib_send_cm_rej(struct ib_cm_id *cm_id, cm_enter_timewait(cm_id_priv); break; default: + pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__, + be32_to_cpu(cm_id_priv->id.local_id), cm_id->state); ret = -EINVAL; goto out; } @@ -2748,6 +2815,9 @@ static int cm_rej_handler(struct cm_work *work) /* fall through */ default: spin_unlock_irq(&cm_id_priv->lock); + pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", + __func__, be32_to_cpu(cm_id_priv->id.local_id), + cm_id_priv->id.state); ret = -EINVAL; goto out; } @@ -2811,6 +2881,9 @@ int ib_send_cm_mra(struct ib_cm_id *cm_id, } /* fall through */ default: + pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", + __func__, be32_to_cpu(cm_id_priv->id.local_id), + cm_id_priv->id.state); ret = -EINVAL; goto error1; } @@ -2912,6 +2985,9 @@ static int cm_mra_handler(struct cm_work *work) counter[CM_MRA_COUNTER]); /* fall through */ default: + pr_debug("%s local_id %d, cm_id_priv->id.state: %d\n", + __func__, be32_to_cpu(cm_id_priv->id.local_id), + cm_id_priv->id.state); goto out; } @@ -3085,6 +3161,12 @@ static int cm_lap_handler(struct cm_work *work) if (!cm_id_priv) return -EINVAL; + ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc, + work->mad_recv_wc->recv_buf.grh, + &cm_id_priv->av); + if (ret) + goto deref; + param = &work->cm_event.param.lap_rcvd; memset(&work->path[0], 0, sizeof(work->path[1])); cm_path_set_rec_type(work->port->cm_dev->ib_device, @@ -3131,9 +3213,6 @@ static int cm_lap_handler(struct cm_work *work) cm_id_priv->id.lap_state = IB_CM_LAP_RCVD; cm_id_priv->tid = lap_msg->hdr.tid; - cm_init_av_for_response(work->port, work->mad_recv_wc->wc, - work->mad_recv_wc->recv_buf.grh, - &cm_id_priv->av); cm_init_av_by_path(param->alternate_path, &cm_id_priv->alt_av, cm_id_priv); ret = atomic_inc_and_test(&cm_id_priv->work_count); @@ -3386,6 +3465,7 @@ static int cm_sidr_req_handler(struct cm_work *work) struct cm_id_private *cm_id_priv, *cur_cm_id_priv; struct cm_sidr_req_msg *sidr_req_msg; struct ib_wc *wc; + int ret; cm_id = ib_create_cm_id(work->port->cm_dev->ib_device, NULL, NULL); if (IS_ERR(cm_id)) @@ -3398,9 +3478,12 @@ static int cm_sidr_req_handler(struct cm_work *work) wc = work->mad_recv_wc->wc; cm_id_priv->av.dgid.global.subnet_prefix = cpu_to_be64(wc->slid); cm_id_priv->av.dgid.global.interface_id = 0; - cm_init_av_for_response(work->port, work->mad_recv_wc->wc, - work->mad_recv_wc->recv_buf.grh, - &cm_id_priv->av); + ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc, + work->mad_recv_wc->recv_buf.grh, + &cm_id_priv->av); + if (ret) + goto out; + cm_id_priv->id.remote_id = sidr_req_msg->request_id; cm_id_priv->tid = sidr_req_msg->hdr.tid; atomic_inc(&cm_id_priv->work_count); @@ -3692,6 +3775,7 @@ static void cm_work_handler(struct work_struct *_work) ret = cm_timewait_handler(work); break; default: + pr_debug("cm_event.event: 0x%x\n", work->cm_event.event); ret = -EINVAL; break; } @@ -3727,6 +3811,8 @@ static int cm_establish(struct ib_cm_id *cm_id) ret = -EISCONN; break; default: + pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__, + be32_to_cpu(cm_id->local_id), cm_id->state); ret = -EINVAL; break; } @@ -3924,6 +4010,9 @@ static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv, ret = 0; break; default: + pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", + __func__, be32_to_cpu(cm_id_priv->id.local_id), + cm_id_priv->id.state); ret = -EINVAL; break; } @@ -3971,6 +4060,9 @@ static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv, ret = 0; break; default: + pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", + __func__, be32_to_cpu(cm_id_priv->id.local_id), + cm_id_priv->id.state); ret = -EINVAL; break; } @@ -4030,6 +4122,9 @@ static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv, ret = 0; break; default: + pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", + __func__, be32_to_cpu(cm_id_priv->id.local_id), + cm_id_priv->id.state); ret = -EINVAL; break; } diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 6294a7001d33..7d38c2bff5ea 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -601,7 +601,7 @@ static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_a int ret; if (addr->sa_family != AF_IB) { - ret = rdma_translate_ip(addr, dev_addr, NULL); + ret = rdma_translate_ip(addr, dev_addr); } else { cma_translate_ib((struct sockaddr_ib *) addr, dev_addr); ret = 0; @@ -2132,7 +2132,7 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id, mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); conn_id->state = RDMA_CM_CONNECT; - ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr, NULL); + ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr); if (ret) { mutex_unlock(&conn_id->handler_mutex); rdma_destroy_id(new_cm_id); @@ -2414,6 +2414,26 @@ out: kfree(work); } +static void cma_init_resolve_route_work(struct cma_work *work, + struct rdma_id_private *id_priv) +{ + work->id = id_priv; + INIT_WORK(&work->work, cma_work_handler); + work->old_state = RDMA_CM_ROUTE_QUERY; + work->new_state = RDMA_CM_ROUTE_RESOLVED; + work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; +} + +static void cma_init_resolve_addr_work(struct cma_work *work, + struct rdma_id_private *id_priv) +{ + work->id = id_priv; + INIT_WORK(&work->work, cma_work_handler); + work->old_state = RDMA_CM_ADDR_QUERY; + work->new_state = RDMA_CM_ADDR_RESOLVED; + work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED; +} + static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms) { struct rdma_route *route = &id_priv->id.route; @@ -2424,11 +2444,7 @@ static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms) if (!work) return -ENOMEM; - work->id = id_priv; - INIT_WORK(&work->work, cma_work_handler); - work->old_state = RDMA_CM_ROUTE_QUERY; - work->new_state = RDMA_CM_ROUTE_RESOLVED; - work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; + cma_init_resolve_route_work(work, id_priv); route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL); if (!route->path_rec) { @@ -2483,11 +2499,7 @@ static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms) if (!work) return -ENOMEM; - work->id = id_priv; - INIT_WORK(&work->work, cma_work_handler); - work->old_state = RDMA_CM_ROUTE_QUERY; - work->new_state = RDMA_CM_ROUTE_RESOLVED; - work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; + cma_init_resolve_route_work(work, id_priv); queue_work(cma_wq, &work->work); return 0; } @@ -2528,8 +2540,10 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv) struct rdma_addr *addr = &route->addr; struct cma_work *work; int ret; - struct net_device *ndev = NULL; - enum ib_gid_type gid_type = IB_GID_TYPE_IB; + struct net_device *ndev; + enum ib_gid_type gid_type = IB_GID_TYPE_ROCE; + unsigned long supported_gids; + u8 default_roce_tos = id_priv->cma_dev->default_roce_tos[id_priv->id.port_num - rdma_start_port(id_priv->cma_dev->device)]; u8 tos = id_priv->tos_set ? id_priv->tos : default_roce_tos; @@ -2539,9 +2553,6 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv) if (!work) return -ENOMEM; - work->id = id_priv; - INIT_WORK(&work->work, cma_work_handler); - route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL); if (!route->path_rec) { ret = -ENOMEM; @@ -2550,30 +2561,29 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv) route->num_paths = 1; - if (addr->dev_addr.bound_dev_if) { - unsigned long supported_gids; - - ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if); - if (!ndev) { - ret = -ENODEV; - goto err2; - } - - supported_gids = roce_gid_type_mask_support(id_priv->id.device, - id_priv->id.port_num); - gid_type = cma_route_gid_type(addr->dev_addr.network, - supported_gids, - id_priv->gid_type); - route->path_rec->rec_type = - sa_conv_gid_to_pathrec_type(gid_type); - sa_path_set_ndev(route->path_rec, &init_net); - sa_path_set_ifindex(route->path_rec, ndev->ifindex); + if (!addr->dev_addr.bound_dev_if) { + ret = -ENODEV; + goto err2; } + + ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if); if (!ndev) { ret = -ENODEV; goto err2; } + supported_gids = roce_gid_type_mask_support(id_priv->id.device, + id_priv->id.port_num); + gid_type = cma_route_gid_type(addr->dev_addr.network, + supported_gids, + id_priv->gid_type); + /* Use the hint from IP Stack to select GID Type */ + if (gid_type < ib_network_to_gid_type(addr->dev_addr.network)) + gid_type = ib_network_to_gid_type(addr->dev_addr.network); + route->path_rec->rec_type = sa_conv_gid_to_pathrec_type(gid_type); + + sa_path_set_ndev(route->path_rec, &init_net); + sa_path_set_ifindex(route->path_rec, ndev->ifindex); sa_path_set_dmac(route->path_rec, addr->dev_addr.dst_dev_addr); rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr, @@ -2581,11 +2591,6 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv) rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.dst_addr, &route->path_rec->dgid); - /* Use the hint from IP Stack to select GID Type */ - if (gid_type < ib_network_to_gid_type(addr->dev_addr.network)) - gid_type = ib_network_to_gid_type(addr->dev_addr.network); - route->path_rec->rec_type = sa_conv_gid_to_pathrec_type(gid_type); - if (((struct sockaddr *)&id_priv->id.route.addr.dst_addr)->sa_family != AF_IB) /* TODO: get the hoplimit from the inet/inet6 device */ route->path_rec->hop_limit = addr->dev_addr.hoplimit; @@ -2607,11 +2612,7 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv) goto err2; } - work->old_state = RDMA_CM_ROUTE_QUERY; - work->new_state = RDMA_CM_ROUTE_RESOLVED; - work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; - work->event.status = 0; - + cma_init_resolve_route_work(work, id_priv); queue_work(cma_wq, &work->work); return 0; @@ -2791,11 +2792,7 @@ static int cma_resolve_loopback(struct rdma_id_private *id_priv) rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid); rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid); - work->id = id_priv; - INIT_WORK(&work->work, cma_work_handler); - work->old_state = RDMA_CM_ADDR_QUERY; - work->new_state = RDMA_CM_ADDR_RESOLVED; - work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED; + cma_init_resolve_addr_work(work, id_priv); queue_work(cma_wq, &work->work); return 0; err: @@ -2821,11 +2818,7 @@ static int cma_resolve_ib_addr(struct rdma_id_private *id_priv) rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, (union ib_gid *) &(((struct sockaddr_ib *) &id_priv->id.route.addr.dst_addr)->sib_addr)); - work->id = id_priv; - INIT_WORK(&work->work, cma_work_handler); - work->old_state = RDMA_CM_ADDR_QUERY; - work->new_state = RDMA_CM_ADDR_RESOLVED; - work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED; + cma_init_resolve_addr_work(work, id_priv); queue_work(cma_wq, &work->work); return 0; err: @@ -3404,9 +3397,10 @@ static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, event.status = ret; break; } - ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num, - id_priv->id.route.path_rec, - &event.param.ud.ah_attr); + ib_init_ah_attr_from_path(id_priv->id.device, + id_priv->id.port_num, + id_priv->id.route.path_rec, + &event.param.ud.ah_attr); event.param.ud.qp_num = rep->qpn; event.param.ud.qkey = rep->qkey; event.event = RDMA_CM_EVENT_ESTABLISHED; @@ -4010,8 +4004,10 @@ static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid, } else if (addr->sa_family == AF_INET6) { memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); } else { - mgid->raw[0] = (gid_type == IB_GID_TYPE_IB) ? 0xff : 0; - mgid->raw[1] = (gid_type == IB_GID_TYPE_IB) ? 0x0e : 0; + mgid->raw[0] = + (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0xff; + mgid->raw[1] = + (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0x0e; mgid->raw[2] = 0; mgid->raw[3] = 0; mgid->raw[4] = 0; @@ -4432,7 +4428,7 @@ static int cma_get_id_stats(struct sk_buff *skb, struct netlink_callback *cb) RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) goto out; if (ibnl_put_attr(skb, nlh, - rdma_addr_size(cma_src_addr(id_priv)), + rdma_addr_size(cma_dst_addr(id_priv)), cma_dst_addr(id_priv), RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) goto out; diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index a1d687a664f8..0deff4c4911b 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -191,13 +191,6 @@ void ib_sa_cleanup(void); int rdma_nl_init(void); void rdma_nl_exit(void); -/** - * Check if there are any listeners to the netlink group - * @group: the netlink group ID - * Returns 0 on success or a negative for no listeners. - */ -int ibnl_chk_listeners(unsigned int group); - int ib_nl_handle_resolve_resp(struct sk_buff *skb, struct nlmsghdr *nlh, struct netlink_ext_ack *extack); @@ -213,11 +206,6 @@ int ib_get_cached_subnet_prefix(struct ib_device *device, u64 *sn_pfx); #ifdef CONFIG_SECURITY_INFINIBAND -int ib_security_pkey_access(struct ib_device *dev, - u8 port_num, - u16 pkey_index, - void *sec); - void ib_security_destroy_port_pkey_list(struct ib_device *device); void ib_security_cache_change(struct ib_device *device, @@ -240,14 +228,6 @@ int ib_mad_agent_security_setup(struct ib_mad_agent *agent, void ib_mad_agent_security_cleanup(struct ib_mad_agent *agent); int ib_mad_enforce_security(struct ib_mad_agent_private *map, u16 pkey_index); #else -static inline int ib_security_pkey_access(struct ib_device *dev, - u8 port_num, - u16 pkey_index, - void *sec) -{ - return 0; -} - static inline void ib_security_destroy_port_pkey_list(struct ib_device *device) { } diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index 30914f3baa5f..21302e077be1 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -1017,32 +1017,22 @@ EXPORT_SYMBOL(ib_modify_port); /** * ib_find_gid - Returns the port number and GID table index where - * a specified GID value occurs. + * a specified GID value occurs. Its searches only for IB link layer. * @device: The device to query. * @gid: The GID value to search for. - * @gid_type: Type of GID. * @ndev: The ndev related to the GID to search for. * @port_num: The port number of the device where the GID value was found. * @index: The index into the GID table where the GID was found. This * parameter may be NULL. */ int ib_find_gid(struct ib_device *device, union ib_gid *gid, - enum ib_gid_type gid_type, struct net_device *ndev, - u8 *port_num, u16 *index) + struct net_device *ndev, u8 *port_num, u16 *index) { union ib_gid tmp_gid; int ret, port, i; for (port = rdma_start_port(device); port <= rdma_end_port(device); ++port) { - if (rdma_cap_roce_gid_table(device, port)) { - if (!ib_find_cached_gid_by_port(device, gid, gid_type, port, - ndev, index)) { - *port_num = port; - return 0; - } - } - - if (gid_type != IB_GID_TYPE_IB) + if (rdma_cap_roce_gid_table(device, port)) continue; for (i = 0; i < device->port_immutable[port].gid_tbl_len; ++i) { diff --git a/drivers/infiniband/core/iwpm_util.c b/drivers/infiniband/core/iwpm_util.c index 3c4faadb8cdd..81528f64061a 100644 --- a/drivers/infiniband/core/iwpm_util.c +++ b/drivers/infiniband/core/iwpm_util.c @@ -654,6 +654,7 @@ int iwpm_send_mapinfo(u8 nl_client, int iwpm_pid) } skb_num++; spin_lock_irqsave(&iwpm_mapinfo_lock, flags); + ret = -EINVAL; for (i = 0; i < IWPM_MAPINFO_HASH_SIZE; i++) { hlist_for_each_entry(map_info, &iwpm_hash_bucket[i], hlist_node) { diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c index cb91245e9163..c50596f7f98a 100644 --- a/drivers/infiniband/core/mad.c +++ b/drivers/infiniband/core/mad.c @@ -49,7 +49,6 @@ #include "smi.h" #include "opa_smi.h" #include "agent.h" -#include "core_priv.h" static int mad_sendq_size = IB_MAD_QP_SEND_SIZE; static int mad_recvq_size = IB_MAD_QP_RECV_SIZE; diff --git a/drivers/infiniband/core/netlink.c b/drivers/infiniband/core/netlink.c index 1fb72c356e36..0c7db94cd9b9 100644 --- a/drivers/infiniband/core/netlink.c +++ b/drivers/infiniband/core/netlink.c @@ -41,8 +41,6 @@ #include <linux/module.h> #include "core_priv.h" -#include "core_priv.h" - static DEFINE_MUTEX(rdma_nl_mutex); static struct sock *nls; static struct { diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c index ab5e1024fea9..8cf15d4a8ac4 100644 --- a/drivers/infiniband/core/sa_query.c +++ b/drivers/infiniband/core/sa_query.c @@ -1227,9 +1227,9 @@ static u8 get_src_path_mask(struct ib_device *device, u8 port_num) return src_path_mask; } -int ib_init_ah_from_path(struct ib_device *device, u8 port_num, - struct sa_path_rec *rec, - struct rdma_ah_attr *ah_attr) +int ib_init_ah_attr_from_path(struct ib_device *device, u8 port_num, + struct sa_path_rec *rec, + struct rdma_ah_attr *ah_attr) { int ret; u16 gid_index; @@ -1341,10 +1341,11 @@ int ib_init_ah_from_path(struct ib_device *device, u8 port_num, return 0; } -EXPORT_SYMBOL(ib_init_ah_from_path); +EXPORT_SYMBOL(ib_init_ah_attr_from_path); static int alloc_mad(struct ib_sa_query *query, gfp_t gfp_mask) { + struct rdma_ah_attr ah_attr; unsigned long flags; spin_lock_irqsave(&query->port->ah_lock, flags); @@ -1356,6 +1357,15 @@ static int alloc_mad(struct ib_sa_query *query, gfp_t gfp_mask) query->sm_ah = query->port->sm_ah; spin_unlock_irqrestore(&query->port->ah_lock, flags); + /* + * Always check if sm_ah has valid dlid assigned, + * before querying for class port info + */ + if ((rdma_query_ah(query->sm_ah->ah, &ah_attr) < 0) || + !rdma_is_valid_unicast_lid(&ah_attr)) { + kref_put(&query->sm_ah->ref, free_sm_ah); + return -EAGAIN; + } query->mad_buf = ib_create_send_mad(query->port->agent, 1, query->sm_ah->pkey_index, 0, IB_MGMT_SA_HDR, IB_MGMT_SA_DATA, diff --git a/drivers/infiniband/core/security.c b/drivers/infiniband/core/security.c index 59b2f96d986a..b61dda6b04fc 100644 --- a/drivers/infiniband/core/security.c +++ b/drivers/infiniband/core/security.c @@ -653,12 +653,11 @@ int ib_security_modify_qp(struct ib_qp *qp, } return ret; } -EXPORT_SYMBOL(ib_security_modify_qp); -int ib_security_pkey_access(struct ib_device *dev, - u8 port_num, - u16 pkey_index, - void *sec) +static int ib_security_pkey_access(struct ib_device *dev, + u8 port_num, + u16 pkey_index, + void *sec) { u64 subnet_prefix; u16 pkey; @@ -678,7 +677,6 @@ int ib_security_pkey_access(struct ib_device *dev, return security_ib_pkey_access(sec, subnet_prefix, pkey); } -EXPORT_SYMBOL(ib_security_pkey_access); static int ib_mad_agent_security_change(struct notifier_block *nb, unsigned long event, diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 21e60b1e2ff4..9a4e899d94b3 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -191,7 +191,7 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, sg_list_start = umem->sg_head.sgl; while (npages) { - ret = get_user_pages(cur_base, + ret = get_user_pages_longterm(cur_base, min_t(unsigned long, npages, PAGE_SIZE / sizeof (struct page *)), gup_flags, page_list, vma_list); @@ -352,7 +352,7 @@ int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, return -EINVAL; } - ret = sg_pcopy_to_buffer(umem->sg_head.sgl, umem->nmap, dst, length, + ret = sg_pcopy_to_buffer(umem->sg_head.sgl, umem->npages, dst, length, offset + ib_umem_offset(umem)); if (ret < 0) diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c index 4b64dd02e090..d4de187b1064 100644 --- a/drivers/infiniband/core/user_mad.c +++ b/drivers/infiniband/core/user_mad.c @@ -233,8 +233,7 @@ static void recv_handler(struct ib_mad_agent *agent, * On OPA devices it is okay to lose the upper 16 bits of LID as this * information is obtained elsewhere. Mask off the upper 16 bits. */ - if (agent->device->port_immutable[agent->port_num].core_cap_flags & - RDMA_CORE_PORT_INTEL_OPA) + if (rdma_cap_opa_mad(agent->device, agent->port_num)) packet->mad.hdr.lid = ib_lid_be16(0xFFFF & mad_recv_wc->wc->slid); else @@ -246,10 +245,14 @@ static void recv_handler(struct ib_mad_agent *agent, if (packet->mad.hdr.grh_present) { struct rdma_ah_attr ah_attr; const struct ib_global_route *grh; + int ret; - ib_init_ah_from_wc(agent->device, agent->port_num, - mad_recv_wc->wc, mad_recv_wc->recv_buf.grh, - &ah_attr); + ret = ib_init_ah_attr_from_wc(agent->device, agent->port_num, + mad_recv_wc->wc, + mad_recv_wc->recv_buf.grh, + &ah_attr); + if (ret) + goto err2; grh = rdma_ah_read_grh(&ah_attr); packet->mad.hdr.gid_index = grh->sgid_index; diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 840b24096690..c216d98bb816 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -1145,10 +1145,7 @@ int ib_uverbs_ex_create_cq(struct ib_uverbs_file *file, min(ucore->inlen, sizeof(cmd)), ib_uverbs_ex_create_cq_cb, NULL); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - return 0; + return PTR_ERR_OR_ZERO(obj); } ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file, diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index e36d27ed4daa..fe72fb303b01 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -421,8 +421,7 @@ static bool find_gid_index(const union ib_gid *gid, const struct ib_gid_attr *gid_attr, void *context) { - struct find_gid_index_context *ctx = - (struct find_gid_index_context *)context; + struct find_gid_index_context *ctx = context; if (ctx->gid_type != gid_attr->gid_type) return false; @@ -481,8 +480,53 @@ int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr, } EXPORT_SYMBOL(ib_get_gids_from_rdma_hdr); +/* Resolve destination mac address and hop limit for unicast destination + * GID entry, considering the source GID entry as well. + * ah_attribute must have have valid port_num, sgid_index. + */ +static int ib_resolve_unicast_gid_dmac(struct ib_device *device, + struct rdma_ah_attr *ah_attr) +{ + struct ib_gid_attr sgid_attr; + struct ib_global_route *grh; + int hop_limit = 0xff; + union ib_gid sgid; + int ret; + + grh = rdma_ah_retrieve_grh(ah_attr); + + ret = ib_query_gid(device, + rdma_ah_get_port_num(ah_attr), + grh->sgid_index, + &sgid, &sgid_attr); + if (ret || !sgid_attr.ndev) { + if (!ret) + ret = -ENXIO; + return ret; + } + + /* If destination is link local and source GID is RoCEv1, + * IP stack is not used. + */ + if (rdma_link_local_addr((struct in6_addr *)grh->dgid.raw) && + sgid_attr.gid_type == IB_GID_TYPE_ROCE) { + rdma_get_ll_mac((struct in6_addr *)grh->dgid.raw, + ah_attr->roce.dmac); + goto done; + } + + ret = rdma_addr_find_l2_eth_by_grh(&sgid, &grh->dgid, + ah_attr->roce.dmac, + sgid_attr.ndev, &hop_limit); +done: + dev_put(sgid_attr.ndev); + + grh->hop_limit = hop_limit; + return ret; +} + /* - * This function creates ah from the incoming packet. + * This function initializes address handle attributes from the incoming packet. * Incoming packet has dgid of the receiver node on which this code is * getting executed and, sgid contains the GID of the sender. * @@ -490,13 +534,10 @@ EXPORT_SYMBOL(ib_get_gids_from_rdma_hdr); * as sgid and, sgid is used as dgid because sgid contains destinations * GID whom to respond to. * - * This is why when calling rdma_addr_find_l2_eth_by_grh() function, the - * position of arguments dgid and sgid do not match the order of the - * parameters. */ -int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, - const struct ib_wc *wc, const struct ib_grh *grh, - struct rdma_ah_attr *ah_attr) +int ib_init_ah_attr_from_wc(struct ib_device *device, u8 port_num, + const struct ib_wc *wc, const struct ib_grh *grh, + struct rdma_ah_attr *ah_attr) { u32 flow_class; u16 gid_index; @@ -523,57 +564,33 @@ int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, if (ret) return ret; + rdma_ah_set_sl(ah_attr, wc->sl); + rdma_ah_set_port_num(ah_attr, port_num); + if (rdma_protocol_roce(device, port_num)) { - int if_index = 0; u16 vlan_id = wc->wc_flags & IB_WC_WITH_VLAN ? wc->vlan_id : 0xffff; - struct net_device *idev; - struct net_device *resolved_dev; if (!(wc->wc_flags & IB_WC_GRH)) return -EPROTOTYPE; - if (!device->get_netdev) - return -EOPNOTSUPP; - - idev = device->get_netdev(device, port_num); - if (!idev) - return -ENODEV; - - ret = rdma_addr_find_l2_eth_by_grh(&dgid, &sgid, - ah_attr->roce.dmac, - wc->wc_flags & IB_WC_WITH_VLAN ? - NULL : &vlan_id, - &if_index, &hoplimit); - if (ret) { - dev_put(idev); - return ret; - } - - resolved_dev = dev_get_by_index(&init_net, if_index); - rcu_read_lock(); - if (resolved_dev != idev && !rdma_is_upper_dev_rcu(idev, - resolved_dev)) - ret = -EHOSTUNREACH; - rcu_read_unlock(); - dev_put(idev); - dev_put(resolved_dev); + ret = get_sgid_index_from_eth(device, port_num, + vlan_id, &dgid, + gid_type, &gid_index); if (ret) return ret; - ret = get_sgid_index_from_eth(device, port_num, vlan_id, - &dgid, gid_type, &gid_index); - if (ret) - return ret; - } - - rdma_ah_set_dlid(ah_attr, wc->slid); - rdma_ah_set_sl(ah_attr, wc->sl); - rdma_ah_set_path_bits(ah_attr, wc->dlid_path_bits); - rdma_ah_set_port_num(ah_attr, port_num); + flow_class = be32_to_cpu(grh->version_tclass_flow); + rdma_ah_set_grh(ah_attr, &sgid, + flow_class & 0xFFFFF, + (u8)gid_index, hoplimit, + (flow_class >> 20) & 0xFF); + return ib_resolve_unicast_gid_dmac(device, ah_attr); + } else { + rdma_ah_set_dlid(ah_attr, wc->slid); + rdma_ah_set_path_bits(ah_attr, wc->dlid_path_bits); - if (wc->wc_flags & IB_WC_GRH) { - if (!rdma_cap_eth_ah(device, port_num)) { + if (wc->wc_flags & IB_WC_GRH) { if (dgid.global.interface_id != cpu_to_be64(IB_SA_WELL_KNOWN_GUID)) { ret = ib_find_cached_gid_by_port(device, &dgid, IB_GID_TYPE_IB, @@ -584,18 +601,17 @@ int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, } else { gid_index = 0; } - } - - flow_class = be32_to_cpu(grh->version_tclass_flow); - rdma_ah_set_grh(ah_attr, &sgid, - flow_class & 0xFFFFF, - (u8)gid_index, hoplimit, - (flow_class >> 20) & 0xFF); + flow_class = be32_to_cpu(grh->version_tclass_flow); + rdma_ah_set_grh(ah_attr, &sgid, + flow_class & 0xFFFFF, + (u8)gid_index, hoplimit, + (flow_class >> 20) & 0xFF); + } + return 0; } - return 0; } -EXPORT_SYMBOL(ib_init_ah_from_wc); +EXPORT_SYMBOL(ib_init_ah_attr_from_wc); struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc, const struct ib_grh *grh, u8 port_num) @@ -603,7 +619,7 @@ struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc, struct rdma_ah_attr ah_attr; int ret; - ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr); + ret = ib_init_ah_attr_from_wc(pd->device, port_num, wc, grh, &ah_attr); if (ret) return ERR_PTR(ret); @@ -1274,11 +1290,6 @@ static int ib_resolve_eth_dmac(struct ib_device *device, grh = rdma_ah_retrieve_grh(ah_attr); - if (rdma_link_local_addr((struct in6_addr *)grh->dgid.raw)) { - rdma_get_ll_mac((struct in6_addr *)grh->dgid.raw, - ah_attr->roce.dmac); - return 0; - } if (rdma_is_multicast_addr((struct in6_addr *)ah_attr->grh.dgid.raw)) { if (ipv6_addr_v4mapped((struct in6_addr *)ah_attr->grh.dgid.raw)) { __be32 addr = 0; @@ -1290,34 +1301,8 @@ static int ib_resolve_eth_dmac(struct ib_device *device, (char *)ah_attr->roce.dmac); } } else { - union ib_gid sgid; - struct ib_gid_attr sgid_attr; - int ifindex; - int hop_limit; - - ret = ib_query_gid(device, - rdma_ah_get_port_num(ah_attr), - grh->sgid_index, - &sgid, &sgid_attr); - - if (ret || !sgid_attr.ndev) { - if (!ret) - ret = -ENXIO; - goto out; - } - - ifindex = sgid_attr.ndev->ifindex; - - ret = - rdma_addr_find_l2_eth_by_grh(&sgid, &grh->dgid, - ah_attr->roce.dmac, - NULL, &ifindex, &hop_limit); - - dev_put(sgid_attr.ndev); - - grh->hop_limit = hop_limit; + ret = ib_resolve_unicast_gid_dmac(device, ah_attr); } -out: return ret; } @@ -1335,6 +1320,7 @@ out: int ib_modify_qp_with_udata(struct ib_qp *qp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { + u8 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port; int ret; if (attr_mask & IB_QP_AV) { @@ -1342,6 +1328,21 @@ int ib_modify_qp_with_udata(struct ib_qp *qp, struct ib_qp_attr *attr, if (ret) return ret; } + + if (rdma_ib_or_roce(qp->device, port)) { + if (attr_mask & IB_QP_RQ_PSN && attr->rq_psn & ~0xffffff) { + pr_warn("%s: %s rq_psn overflow, masking to 24 bits\n", + __func__, qp->device->name); + attr->rq_psn &= 0xffffff; + } + + if (attr_mask & IB_QP_SQ_PSN && attr->sq_psn & ~0xffffff) { + pr_warn("%s: %s sq_psn overflow, masking to 24 bits\n", + __func__, qp->device->name); + attr->sq_psn &= 0xffffff; + } + } + ret = ib_security_modify_qp(qp, attr, attr_mask, udata); if (!ret && (attr_mask & IB_QP_PORT)) qp->port = attr->port_num; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c index 61764f7aa79b..eb7195c20b88 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c @@ -410,7 +410,6 @@ int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, if (rc) { dev_err(&nq->pdev->dev, "Failed to request IRQ for NQ: %#x", rc); - bnxt_qplib_disable_nq(nq); goto fail; } diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c index 21db3b48a617..844c9e78df8b 100644 --- a/drivers/infiniband/hw/cxgb4/cm.c +++ b/drivers/infiniband/hw/cxgb4/cm.c @@ -4097,9 +4097,15 @@ static void process_work(struct work_struct *work) dev = *((struct c4iw_dev **) (skb->cb + sizeof(void *))); opcode = rpl->ot.opcode; - ret = work_handlers[opcode](dev, skb); - if (!ret) + if (opcode >= ARRAY_SIZE(work_handlers) || + !work_handlers[opcode]) { + pr_err("No handler for opcode 0x%x.\n", opcode); kfree_skb(skb); + } else { + ret = work_handlers[opcode](dev, skb); + if (!ret) + kfree_skb(skb); + } process_timedout_eps(); } } diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c index af77d128d242..7a9d0de89d6a 100644 --- a/drivers/infiniband/hw/cxgb4/device.c +++ b/drivers/infiniband/hw/cxgb4/device.c @@ -66,7 +66,7 @@ MODULE_PARM_DESC(c4iw_wr_log_size_order, static LIST_HEAD(uld_ctx_list); static DEFINE_MUTEX(dev_mutex); -struct workqueue_struct *reg_workq; +static struct workqueue_struct *reg_workq; #define DB_FC_RESUME_SIZE 64 #define DB_FC_RESUME_DELAY 1 @@ -108,19 +108,19 @@ void c4iw_log_wr_stats(struct t4_wq *wq, struct t4_cqe *cqe) idx = (atomic_inc_return(&wq->rdev->wr_log_idx) - 1) & (wq->rdev->wr_log_size - 1); le.poll_sge_ts = cxgb4_read_sge_timestamp(wq->rdev->lldi.ports[0]); - getnstimeofday(&le.poll_host_ts); + le.poll_host_time = ktime_get(); le.valid = 1; le.cqe_sge_ts = CQE_TS(cqe); if (SQ_TYPE(cqe)) { le.qid = wq->sq.qid; le.opcode = CQE_OPCODE(cqe); - le.post_host_ts = wq->sq.sw_sq[wq->sq.cidx].host_ts; + le.post_host_time = wq->sq.sw_sq[wq->sq.cidx].host_time; le.post_sge_ts = wq->sq.sw_sq[wq->sq.cidx].sge_ts; le.wr_id = CQE_WRID_SQ_IDX(cqe); } else { le.qid = wq->rq.qid; le.opcode = FW_RI_RECEIVE; - le.post_host_ts = wq->rq.sw_rq[wq->rq.cidx].host_ts; + le.post_host_time = wq->rq.sw_rq[wq->rq.cidx].host_time; le.post_sge_ts = wq->rq.sw_rq[wq->rq.cidx].sge_ts; le.wr_id = CQE_WRID_MSN(cqe); } @@ -130,9 +130,9 @@ void c4iw_log_wr_stats(struct t4_wq *wq, struct t4_cqe *cqe) static int wr_log_show(struct seq_file *seq, void *v) { struct c4iw_dev *dev = seq->private; - struct timespec prev_ts = {0, 0}; + ktime_t prev_time; struct wr_log_entry *lep; - int prev_ts_set = 0; + int prev_time_set = 0; int idx, end; #define ts2ns(ts) div64_u64((ts) * dev->rdev.lldi.cclk_ps, 1000) @@ -145,33 +145,29 @@ static int wr_log_show(struct seq_file *seq, void *v) lep = &dev->rdev.wr_log[idx]; while (idx != end) { if (lep->valid) { - if (!prev_ts_set) { - prev_ts_set = 1; - prev_ts = lep->poll_host_ts; + if (!prev_time_set) { + prev_time_set = 1; + prev_time = lep->poll_host_time; } - seq_printf(seq, "%04u: sec %lu nsec %lu qid %u opcode " - "%u %s 0x%x host_wr_delta sec %lu nsec %lu " + seq_printf(seq, "%04u: nsec %llu qid %u opcode " + "%u %s 0x%x host_wr_delta nsec %llu " "post_sge_ts 0x%llx cqe_sge_ts 0x%llx " "poll_sge_ts 0x%llx post_poll_delta_ns %llu " "cqe_poll_delta_ns %llu\n", idx, - timespec_sub(lep->poll_host_ts, - prev_ts).tv_sec, - timespec_sub(lep->poll_host_ts, - prev_ts).tv_nsec, + ktime_to_ns(ktime_sub(lep->poll_host_time, + prev_time)), lep->qid, lep->opcode, lep->opcode == FW_RI_RECEIVE ? "msn" : "wrid", lep->wr_id, - timespec_sub(lep->poll_host_ts, - lep->post_host_ts).tv_sec, - timespec_sub(lep->poll_host_ts, - lep->post_host_ts).tv_nsec, + ktime_to_ns(ktime_sub(lep->poll_host_time, + lep->post_host_time)), lep->post_sge_ts, lep->cqe_sge_ts, lep->poll_sge_ts, ts2ns(lep->poll_sge_ts - lep->post_sge_ts), ts2ns(lep->poll_sge_ts - lep->cqe_sge_ts)); - prev_ts = lep->poll_host_ts; + prev_time = lep->poll_host_time; } idx++; if (idx > (dev->rdev.wr_log_size - 1)) diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h index 65dd3726ca02..cc929002c05e 100644 --- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h +++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h @@ -153,8 +153,8 @@ struct c4iw_hw_queue { }; struct wr_log_entry { - struct timespec post_host_ts; - struct timespec poll_host_ts; + ktime_t post_host_time; + ktime_t poll_host_time; u64 post_sge_ts; u64 cqe_sge_ts; u64 poll_sge_ts; diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c index d5c92fc520d6..de77b6027d69 100644 --- a/drivers/infiniband/hw/cxgb4/qp.c +++ b/drivers/infiniband/hw/cxgb4/qp.c @@ -1042,7 +1042,7 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, if (c4iw_wr_log) { swsqe->sge_ts = cxgb4_read_sge_timestamp( qhp->rhp->rdev.lldi.ports[0]); - getnstimeofday(&swsqe->host_ts); + swsqe->host_time = ktime_get(); } init_wr_hdr(wqe, qhp->wq.sq.pidx, fw_opcode, fw_flags, len16); @@ -1117,8 +1117,8 @@ int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].sge_ts = cxgb4_read_sge_timestamp( qhp->rhp->rdev.lldi.ports[0]); - getnstimeofday( - &qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].host_ts); + qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].host_time = + ktime_get(); } wqe->recv.opcode = FW_RI_RECV_WR; diff --git a/drivers/infiniband/hw/cxgb4/t4.h b/drivers/infiniband/hw/cxgb4/t4.h index 79e8ee12c391..8369c7c8de83 100644 --- a/drivers/infiniband/hw/cxgb4/t4.h +++ b/drivers/infiniband/hw/cxgb4/t4.h @@ -277,7 +277,7 @@ struct t4_swsqe { int signaled; u16 idx; int flushed; - struct timespec host_ts; + ktime_t host_time; u64 sge_ts; }; @@ -318,7 +318,7 @@ struct t4_sq { struct t4_swrqe { u64 wr_id; - struct timespec host_ts; + ktime_t host_time; u64 sge_ts; }; diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h index 8ce9118d4a7f..99921acd2ab4 100644 --- a/drivers/infiniband/hw/hfi1/hfi.h +++ b/drivers/infiniband/hw/hfi1/hfi.h @@ -1623,7 +1623,7 @@ static int ingress_pkey_table_search(struct hfi1_pportdata *ppd, u16 pkey) * the 'error info' for this failure. */ static void ingress_pkey_table_fail(struct hfi1_pportdata *ppd, u16 pkey, - u16 slid) + u32 slid) { struct hfi1_devdata *dd = ppd->dd; diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c index cf8dba34fe30..34547a48a445 100644 --- a/drivers/infiniband/hw/hfi1/mad.c +++ b/drivers/infiniband/hw/hfi1/mad.c @@ -4348,11 +4348,7 @@ static int opa_local_smp_check(struct hfi1_ibport *ibp, */ if (pkey == LIM_MGMT_P_KEY || pkey == FULL_MGMT_P_KEY) return 0; - /* - * On OPA devices it is okay to lose the upper 16 bits of LID as this - * information is obtained elsewhere. Mask off the upper 16 bits. - */ - ingress_pkey_table_fail(ppd, pkey, ib_lid_cpu16(0xFFFF & in_wc->slid)); + ingress_pkey_table_fail(ppd, pkey, in_wc->slid); return 1; } diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c index af5f7936f7e5..68d5c3cce2eb 100644 --- a/drivers/infiniband/hw/hfi1/rc.c +++ b/drivers/infiniband/hw/hfi1/rc.c @@ -843,11 +843,11 @@ static inline void hfi1_make_rc_ack_16B(struct rvt_qp *qp, /* Convert dwords to flits */ len = (*hwords + *nwords) >> 1; - hfi1_make_16b_hdr(hdr, - ppd->lid | rdma_ah_get_path_bits(&qp->remote_ah_attr), + hfi1_make_16b_hdr(hdr, ppd->lid | + (rdma_ah_get_path_bits(&qp->remote_ah_attr) & + ((1 << ppd->lmc) - 1)), opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), - 16B), - len, pkey, becn, 0, l4, sc5); + 16B), len, pkey, becn, 0, l4, sc5); bth0 = pkey | (OP(ACKNOWLEDGE) << 24); bth0 |= extra_bytes << 20; diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c index a38785e224cc..6d27c8594b34 100644 --- a/drivers/infiniband/hw/hfi1/verbs.c +++ b/drivers/infiniband/hw/hfi1/verbs.c @@ -1486,7 +1486,7 @@ static int query_port(struct rvt_dev_info *rdi, u8 port_num, props->max_mtu = mtu_to_enum((!valid_ib_mtu(hfi1_max_mtu) ? 4096 : hfi1_max_mtu), IB_MTU_4096); props->active_mtu = !valid_ib_mtu(ppd->ibmtu) ? props->max_mtu : - mtu_to_enum(ppd->ibmtu, IB_MTU_2048); + mtu_to_enum(ppd->ibmtu, IB_MTU_4096); /* * sm_lid of 0xFFFF needs special handling so that it can diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile index ff426a625e13..97bf2cd1cacb 100644 --- a/drivers/infiniband/hw/hns/Makefile +++ b/drivers/infiniband/hw/hns/Makefile @@ -5,7 +5,7 @@ ccflags-y := -Idrivers/net/ethernet/hisilicon/hns3 obj-$(CONFIG_INFINIBAND_HNS) += hns-roce.o -hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_eq.o hns_roce_pd.o \ +hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \ hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \ hns_roce_cq.o hns_roce_alloc.o obj-$(CONFIG_INFINIBAND_HNS_HIP06) += hns-roce-hw-v1.o diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.c b/drivers/infiniband/hw/hns/hns_roce_cmd.c index 1085cb249bc1..9ebe839d8b24 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cmd.c +++ b/drivers/infiniband/hw/hns/hns_roce_cmd.c @@ -103,6 +103,7 @@ void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status, context->out_param = out_param; complete(&context->done); } +EXPORT_SYMBOL_GPL(hns_roce_cmd_event); /* this should be called with "use_events" */ static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.h b/drivers/infiniband/hw/hns/hns_roce_cmd.h index b1c94223c28b..9549ae51a0dd 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cmd.h +++ b/drivers/infiniband/hw/hns/hns_roce_cmd.h @@ -88,6 +88,16 @@ enum { HNS_ROCE_CMD_DESTROY_SRQC_BT0 = 0x38, HNS_ROCE_CMD_DESTROY_SRQC_BT1 = 0x39, HNS_ROCE_CMD_DESTROY_SRQC_BT2 = 0x3a, + + /* EQC commands */ + HNS_ROCE_CMD_CREATE_AEQC = 0x80, + HNS_ROCE_CMD_MODIFY_AEQC = 0x81, + HNS_ROCE_CMD_QUERY_AEQC = 0x82, + HNS_ROCE_CMD_DESTROY_AEQC = 0x83, + HNS_ROCE_CMD_CREATE_CEQC = 0x90, + HNS_ROCE_CMD_MODIFY_CEQC = 0x91, + HNS_ROCE_CMD_QUERY_CEQC = 0x92, + HNS_ROCE_CMD_DESTROY_CEQC = 0x93, }; enum { diff --git a/drivers/infiniband/hw/hns/hns_roce_common.h b/drivers/infiniband/hw/hns/hns_roce_common.h index 7ecb7a4147a8..dd67fafd0c40 100644 --- a/drivers/infiniband/hw/hns/hns_roce_common.h +++ b/drivers/infiniband/hw/hns/hns_roce_common.h @@ -376,6 +376,12 @@ #define ROCEE_RX_CMQ_TAIL_REG 0x07024 #define ROCEE_RX_CMQ_HEAD_REG 0x07028 +#define ROCEE_VF_MB_CFG0_REG 0x40 +#define ROCEE_VF_MB_STATUS_REG 0x58 + +#define ROCEE_VF_EQ_DB_CFG0_REG 0x238 +#define ROCEE_VF_EQ_DB_CFG1_REG 0x23C + #define ROCEE_VF_SMAC_CFG0_REG 0x12000 #define ROCEE_VF_SMAC_CFG1_REG 0x12004 @@ -385,4 +391,9 @@ #define ROCEE_VF_SGID_CFG3_REG 0x1000c #define ROCEE_VF_SGID_CFG4_REG 0x10010 +#define ROCEE_VF_ABN_INT_CFG_REG 0x13000 +#define ROCEE_VF_ABN_INT_ST_REG 0x13004 +#define ROCEE_VF_ABN_INT_EN_REG 0x13008 +#define ROCEE_VF_EVENT_INT_EN_REG 0x1300c + #endif /* _HNS_ROCE_COMMON_H */ diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 2111b57a3489..bccc9b54c9ce 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -196,15 +196,14 @@ void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) if (ret) dev_err(dev, "HW2SW_CQ failed (%d) for CQN %06lx\n", ret, hr_cq->cqn); - if (hr_dev->eq_table.eq) { - /* Waiting interrupt process procedure carried out */ - synchronize_irq(hr_dev->eq_table.eq[hr_cq->vector].irq); - - /* wait for all interrupt processed */ - if (atomic_dec_and_test(&hr_cq->refcount)) - complete(&hr_cq->free); - wait_for_completion(&hr_cq->free); - } + + /* Waiting interrupt process procedure carried out */ + synchronize_irq(hr_dev->eq_table.eq[hr_cq->vector].irq); + + /* wait for all interrupt processed */ + if (atomic_dec_and_test(&hr_cq->refcount)) + complete(&hr_cq->free); + wait_for_completion(&hr_cq->free); spin_lock_irq(&cq_table->lock); radix_tree_delete(&cq_table->tree, hr_cq->cqn); @@ -460,6 +459,7 @@ void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn) ++cq->arm_sn; cq->comp(cq); } +EXPORT_SYMBOL_GPL(hns_roce_cq_completion); void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type) { @@ -482,6 +482,7 @@ void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type) if (atomic_dec_and_test(&cq->refcount)) complete(&cq->free); } +EXPORT_SYMBOL_GPL(hns_roce_cq_event); int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev) { diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index b154ce40cded..dcfd2099bac1 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -62,12 +62,16 @@ #define HNS_ROCE_CQE_WCMD_EMPTY_BIT 0x2 #define HNS_ROCE_MIN_CQE_CNT 16 -#define HNS_ROCE_MAX_IRQ_NUM 34 +#define HNS_ROCE_MAX_IRQ_NUM 128 -#define HNS_ROCE_COMP_VEC_NUM 32 +#define EQ_ENABLE 1 +#define EQ_DISABLE 0 -#define HNS_ROCE_AEQE_VEC_NUM 1 -#define HNS_ROCE_AEQE_OF_VEC_NUM 1 +#define HNS_ROCE_CEQ 0 +#define HNS_ROCE_AEQ 1 + +#define HNS_ROCE_CEQ_ENTRY_SIZE 0x4 +#define HNS_ROCE_AEQ_ENTRY_SIZE 0x10 /* 4G/4K = 1M */ #define HNS_ROCE_SL_SHIFT 28 @@ -130,6 +134,7 @@ enum hns_roce_event { HNS_ROCE_EVENT_TYPE_DB_OVERFLOW = 0x12, HNS_ROCE_EVENT_TYPE_MB = 0x13, HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW = 0x14, + HNS_ROCE_EVENT_TYPE_FLR = 0x15, }; /* Local Work Queue Catastrophic Error,SUBTYPE 0x5 */ @@ -485,6 +490,45 @@ struct hns_roce_ib_iboe { u8 phy_port[HNS_ROCE_MAX_PORTS]; }; +enum { + HNS_ROCE_EQ_STAT_INVALID = 0, + HNS_ROCE_EQ_STAT_VALID = 2, +}; + +struct hns_roce_ceqe { + u32 comp; +}; + +struct hns_roce_aeqe { + u32 asyn; + union { + struct { + u32 qp; + u32 rsv0; + u32 rsv1; + } qp_event; + + struct { + u32 cq; + u32 rsv0; + u32 rsv1; + } cq_event; + + struct { + u32 ceqe; + u32 rsv0; + u32 rsv1; + } ce_event; + + struct { + __le64 out_param; + __le16 token; + u8 status; + u8 rsv0; + } __packed cmd; + } event; +}; + struct hns_roce_eq { struct hns_roce_dev *hr_dev; void __iomem *doorbell; @@ -498,11 +542,31 @@ struct hns_roce_eq { int log_page_size; int cons_index; struct hns_roce_buf_list *buf_list; + int over_ignore; + int coalesce; + int arm_st; + u64 eqe_ba; + int eqe_ba_pg_sz; + int eqe_buf_pg_sz; + int hop_num; + u64 *bt_l0; /* Base address table for L0 */ + u64 **bt_l1; /* Base address table for L1 */ + u64 **buf; + dma_addr_t l0_dma; + dma_addr_t *l1_dma; + dma_addr_t *buf_dma; + u32 l0_last_num; /* L0 last chunk num */ + u32 l1_last_num; /* L1 last chunk num */ + int eq_max_cnt; + int eq_period; + int shift; + dma_addr_t cur_eqe_ba; + dma_addr_t nxt_eqe_ba; }; struct hns_roce_eq_table { struct hns_roce_eq *eq; - void __iomem **eqc_base; + void __iomem **eqc_base; /* only for hw v1 */ }; struct hns_roce_caps { @@ -528,7 +592,7 @@ struct hns_roce_caps { u32 min_wqes; int reserved_cqs; int num_aeq_vectors; /* 1 */ - int num_comp_vectors; /* 32 ceq */ + int num_comp_vectors; int num_other_vectors; int num_mtpts; u32 num_mtt_segs; @@ -550,7 +614,7 @@ struct hns_roce_caps { u32 pbl_buf_pg_sz; u32 pbl_hop_num; int aeqe_depth; - int ceqe_depth[HNS_ROCE_COMP_VEC_NUM]; + int ceqe_depth; enum ib_mtu max_mtu; u32 qpc_bt_num; u32 srqc_bt_num; @@ -574,6 +638,9 @@ struct hns_roce_caps { u32 cqe_ba_pg_sz; u32 cqe_buf_pg_sz; u32 cqe_hop_num; + u32 eqe_ba_pg_sz; + u32 eqe_buf_pg_sz; + u32 eqe_hop_num; u32 chunk_sz; /* chunk size in non multihop mode*/ u64 flags; }; @@ -623,6 +690,8 @@ struct hns_roce_hw { int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr); int (*destroy_cq)(struct ib_cq *ibcq); int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period); + int (*init_eq)(struct hns_roce_dev *hr_dev); + void (*cleanup_eq)(struct hns_roce_dev *hr_dev); }; struct hns_roce_dev { diff --git a/drivers/infiniband/hw/hns/hns_roce_eq.c b/drivers/infiniband/hw/hns/hns_roce_eq.c deleted file mode 100644 index d184431e2bf5..000000000000 --- a/drivers/infiniband/hw/hns/hns_roce_eq.c +++ /dev/null @@ -1,759 +0,0 @@ -/* - * Copyright (c) 2016 Hisilicon Limited. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include <linux/platform_device.h> -#include <linux/interrupt.h> -#include "hns_roce_common.h" -#include "hns_roce_device.h" -#include "hns_roce_eq.h" - -static void eq_set_cons_index(struct hns_roce_eq *eq, int req_not) -{ - roce_raw_write((eq->cons_index & CONS_INDEX_MASK) | - (req_not << eq->log_entries), eq->doorbell); - /* Memory barrier */ - mb(); -} - -static struct hns_roce_aeqe *get_aeqe(struct hns_roce_eq *eq, u32 entry) -{ - unsigned long off = (entry & (eq->entries - 1)) * - HNS_ROCE_AEQ_ENTRY_SIZE; - - return (struct hns_roce_aeqe *)((u8 *) - (eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) + - off % HNS_ROCE_BA_SIZE); -} - -static struct hns_roce_aeqe *next_aeqe_sw(struct hns_roce_eq *eq) -{ - struct hns_roce_aeqe *aeqe = get_aeqe(eq, eq->cons_index); - - return (roce_get_bit(aeqe->asyn, HNS_ROCE_AEQE_U32_4_OWNER_S) ^ - !!(eq->cons_index & eq->entries)) ? aeqe : NULL; -} - -static void hns_roce_wq_catas_err_handle(struct hns_roce_dev *hr_dev, - struct hns_roce_aeqe *aeqe, int qpn) -{ - struct device *dev = &hr_dev->pdev->dev; - - dev_warn(dev, "Local Work Queue Catastrophic Error.\n"); - switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M, - HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) { - case HNS_ROCE_LWQCE_QPC_ERROR: - dev_warn(dev, "QP %d, QPC error.\n", qpn); - break; - case HNS_ROCE_LWQCE_MTU_ERROR: - dev_warn(dev, "QP %d, MTU error.\n", qpn); - break; - case HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR: - dev_warn(dev, "QP %d, WQE BA addr error.\n", qpn); - break; - case HNS_ROCE_LWQCE_WQE_ADDR_ERROR: - dev_warn(dev, "QP %d, WQE addr error.\n", qpn); - break; - case HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR: - dev_warn(dev, "QP %d, WQE shift error\n", qpn); - break; - case HNS_ROCE_LWQCE_SL_ERROR: - dev_warn(dev, "QP %d, SL error.\n", qpn); - break; - case HNS_ROCE_LWQCE_PORT_ERROR: - dev_warn(dev, "QP %d, port error.\n", qpn); - break; - default: - break; - } -} - -static void hns_roce_local_wq_access_err_handle(struct hns_roce_dev *hr_dev, - struct hns_roce_aeqe *aeqe, - int qpn) -{ - struct device *dev = &hr_dev->pdev->dev; - - dev_warn(dev, "Local Access Violation Work Queue Error.\n"); - switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M, - HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) { - case HNS_ROCE_LAVWQE_R_KEY_VIOLATION: - dev_warn(dev, "QP %d, R_key violation.\n", qpn); - break; - case HNS_ROCE_LAVWQE_LENGTH_ERROR: - dev_warn(dev, "QP %d, length error.\n", qpn); - break; - case HNS_ROCE_LAVWQE_VA_ERROR: - dev_warn(dev, "QP %d, VA error.\n", qpn); - break; - case HNS_ROCE_LAVWQE_PD_ERROR: - dev_err(dev, "QP %d, PD error.\n", qpn); - break; - case HNS_ROCE_LAVWQE_RW_ACC_ERROR: - dev_warn(dev, "QP %d, rw acc error.\n", qpn); - break; - case HNS_ROCE_LAVWQE_KEY_STATE_ERROR: - dev_warn(dev, "QP %d, key state error.\n", qpn); - break; - case HNS_ROCE_LAVWQE_MR_OPERATION_ERROR: - dev_warn(dev, "QP %d, MR operation error.\n", qpn); - break; - default: - break; - } -} - -static void hns_roce_qp_err_handle(struct hns_roce_dev *hr_dev, - struct hns_roce_aeqe *aeqe, - int event_type) -{ - struct device *dev = &hr_dev->pdev->dev; - int phy_port; - int qpn; - - qpn = roce_get_field(aeqe->event.qp_event.qp, - HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M, - HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S); - phy_port = roce_get_field(aeqe->event.qp_event.qp, - HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M, - HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S); - if (qpn <= 1) - qpn = HNS_ROCE_MAX_PORTS * qpn + phy_port; - - switch (event_type) { - case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: - dev_warn(dev, "Invalid Req Local Work Queue Error.\n" - "QP %d, phy_port %d.\n", qpn, phy_port); - break; - case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: - hns_roce_wq_catas_err_handle(hr_dev, aeqe, qpn); - break; - case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: - hns_roce_local_wq_access_err_handle(hr_dev, aeqe, qpn); - break; - default: - break; - } - - hns_roce_qp_event(hr_dev, qpn, event_type); -} - -static void hns_roce_cq_err_handle(struct hns_roce_dev *hr_dev, - struct hns_roce_aeqe *aeqe, - int event_type) -{ - struct device *dev = &hr_dev->pdev->dev; - u32 cqn; - - cqn = le32_to_cpu(roce_get_field(aeqe->event.cq_event.cq, - HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M, - HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S)); - - switch (event_type) { - case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: - dev_warn(dev, "CQ 0x%x access err.\n", cqn); - break; - case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: - dev_warn(dev, "CQ 0x%x overflow\n", cqn); - break; - case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID: - dev_warn(dev, "CQ 0x%x ID invalid.\n", cqn); - break; - default: - break; - } - - hns_roce_cq_event(hr_dev, cqn, event_type); -} - -static void hns_roce_db_overflow_handle(struct hns_roce_dev *hr_dev, - struct hns_roce_aeqe *aeqe) -{ - struct device *dev = &hr_dev->pdev->dev; - - switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M, - HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) { - case HNS_ROCE_DB_SUBTYPE_SDB_OVF: - dev_warn(dev, "SDB overflow.\n"); - break; - case HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF: - dev_warn(dev, "SDB almost overflow.\n"); - break; - case HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP: - dev_warn(dev, "SDB almost empty.\n"); - break; - case HNS_ROCE_DB_SUBTYPE_ODB_OVF: - dev_warn(dev, "ODB overflow.\n"); - break; - case HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF: - dev_warn(dev, "ODB almost overflow.\n"); - break; - case HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP: - dev_warn(dev, "SDB almost empty.\n"); - break; - default: - break; - } -} - -static int hns_roce_aeq_int(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) -{ - struct device *dev = &hr_dev->pdev->dev; - struct hns_roce_aeqe *aeqe; - int aeqes_found = 0; - int event_type; - - while ((aeqe = next_aeqe_sw(eq))) { - dev_dbg(dev, "aeqe = %p, aeqe->asyn.event_type = 0x%lx\n", aeqe, - roce_get_field(aeqe->asyn, - HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M, - HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S)); - /* Memory barrier */ - rmb(); - - event_type = roce_get_field(aeqe->asyn, - HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M, - HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S); - switch (event_type) { - case HNS_ROCE_EVENT_TYPE_PATH_MIG: - dev_warn(dev, "PATH MIG not supported\n"); - break; - case HNS_ROCE_EVENT_TYPE_COMM_EST: - dev_warn(dev, "COMMUNICATION established\n"); - break; - case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: - dev_warn(dev, "SQ DRAINED not supported\n"); - break; - case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: - dev_warn(dev, "PATH MIG failed\n"); - break; - case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: - case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: - case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: - hns_roce_qp_err_handle(hr_dev, aeqe, event_type); - break; - case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: - case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: - case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: - dev_warn(dev, "SRQ not support!\n"); - break; - case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: - case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: - case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID: - hns_roce_cq_err_handle(hr_dev, aeqe, event_type); - break; - case HNS_ROCE_EVENT_TYPE_PORT_CHANGE: - dev_warn(dev, "port change.\n"); - break; - case HNS_ROCE_EVENT_TYPE_MB: - hns_roce_cmd_event(hr_dev, - le16_to_cpu(aeqe->event.cmd.token), - aeqe->event.cmd.status, - le64_to_cpu(aeqe->event.cmd.out_param - )); - break; - case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW: - hns_roce_db_overflow_handle(hr_dev, aeqe); - break; - case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW: - dev_warn(dev, "CEQ 0x%lx overflow.\n", - roce_get_field(aeqe->event.ce_event.ceqe, - HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M, - HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S)); - break; - default: - dev_warn(dev, "Unhandled event %d on EQ %d at index %u\n", - event_type, eq->eqn, eq->cons_index); - break; - } - - eq->cons_index++; - aeqes_found = 1; - - if (eq->cons_index > 2 * hr_dev->caps.aeqe_depth - 1) { - dev_warn(dev, "cons_index overflow, set back to zero\n" - ); - eq->cons_index = 0; - } - } - - eq_set_cons_index(eq, 0); - - return aeqes_found; -} - -static struct hns_roce_ceqe *get_ceqe(struct hns_roce_eq *eq, u32 entry) -{ - unsigned long off = (entry & (eq->entries - 1)) * - HNS_ROCE_CEQ_ENTRY_SIZE; - - return (struct hns_roce_ceqe *)((u8 *) - (eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) + - off % HNS_ROCE_BA_SIZE); -} - -static struct hns_roce_ceqe *next_ceqe_sw(struct hns_roce_eq *eq) -{ - struct hns_roce_ceqe *ceqe = get_ceqe(eq, eq->cons_index); - - return (!!(roce_get_bit(ceqe->ceqe.comp, - HNS_ROCE_CEQE_CEQE_COMP_OWNER_S))) ^ - (!!(eq->cons_index & eq->entries)) ? ceqe : NULL; -} - -static int hns_roce_ceq_int(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) -{ - struct hns_roce_ceqe *ceqe; - int ceqes_found = 0; - u32 cqn; - - while ((ceqe = next_ceqe_sw(eq))) { - /* Memory barrier */ - rmb(); - cqn = roce_get_field(ceqe->ceqe.comp, - HNS_ROCE_CEQE_CEQE_COMP_CQN_M, - HNS_ROCE_CEQE_CEQE_COMP_CQN_S); - hns_roce_cq_completion(hr_dev, cqn); - - ++eq->cons_index; - ceqes_found = 1; - - if (eq->cons_index > 2 * hr_dev->caps.ceqe_depth[eq->eqn] - 1) { - dev_warn(&eq->hr_dev->pdev->dev, - "cons_index overflow, set back to zero\n"); - eq->cons_index = 0; - } - } - - eq_set_cons_index(eq, 0); - - return ceqes_found; -} - -static int hns_roce_aeq_ovf_int(struct hns_roce_dev *hr_dev, - struct hns_roce_eq *eq) -{ - struct device *dev = &eq->hr_dev->pdev->dev; - int eqovf_found = 0; - u32 caepaemask_val; - u32 cealmovf_val; - u32 caepaest_val; - u32 aeshift_val; - u32 ceshift_val; - u32 cemask_val; - int i = 0; - - /** - * AEQ overflow ECC mult bit err CEQ overflow alarm - * must clear interrupt, mask irq, clear irq, cancel mask operation - */ - aeshift_val = roce_read(hr_dev, ROCEE_CAEP_AEQC_AEQE_SHIFT_REG); - - if (roce_get_bit(aeshift_val, - ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQ_ALM_OVF_INT_ST_S) == 1) { - dev_warn(dev, "AEQ overflow!\n"); - - /* Set mask */ - caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG); - roce_set_bit(caepaemask_val, - ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S, - HNS_ROCE_INT_MASK_ENABLE); - roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val); - - /* Clear int state(INT_WC : write 1 clear) */ - caepaest_val = roce_read(hr_dev, ROCEE_CAEP_AE_ST_REG); - roce_set_bit(caepaest_val, - ROCEE_CAEP_AE_ST_CAEP_AEQ_ALM_OVF_S, 1); - roce_write(hr_dev, ROCEE_CAEP_AE_ST_REG, caepaest_val); - - /* Clear mask */ - caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG); - roce_set_bit(caepaemask_val, - ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S, - HNS_ROCE_INT_MASK_DISABLE); - roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val); - } - - /* CEQ almost overflow */ - for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) { - ceshift_val = roce_read(hr_dev, ROCEE_CAEP_CEQC_SHIFT_0_REG + - i * CEQ_REG_OFFSET); - - if (roce_get_bit(ceshift_val, - ROCEE_CAEP_CEQC_SHIFT_CAEP_CEQ_ALM_OVF_INT_ST_S) == 1) { - dev_warn(dev, "CEQ[%d] almost overflow!\n", i); - eqovf_found++; - - /* Set mask */ - cemask_val = roce_read(hr_dev, - ROCEE_CAEP_CE_IRQ_MASK_0_REG + - i * CEQ_REG_OFFSET); - roce_set_bit(cemask_val, - ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S, - HNS_ROCE_INT_MASK_ENABLE); - roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG + - i * CEQ_REG_OFFSET, cemask_val); - - /* Clear int state(INT_WC : write 1 clear) */ - cealmovf_val = roce_read(hr_dev, - ROCEE_CAEP_CEQ_ALM_OVF_0_REG + - i * CEQ_REG_OFFSET); - roce_set_bit(cealmovf_val, - ROCEE_CAEP_CEQ_ALM_OVF_CAEP_CEQ_ALM_OVF_S, - 1); - roce_write(hr_dev, ROCEE_CAEP_CEQ_ALM_OVF_0_REG + - i * CEQ_REG_OFFSET, cealmovf_val); - - /* Clear mask */ - cemask_val = roce_read(hr_dev, - ROCEE_CAEP_CE_IRQ_MASK_0_REG + - i * CEQ_REG_OFFSET); - roce_set_bit(cemask_val, - ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S, - HNS_ROCE_INT_MASK_DISABLE); - roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG + - i * CEQ_REG_OFFSET, cemask_val); - } - } - - /* ECC multi-bit error alarm */ - dev_warn(dev, "ECC UCERR ALARM: 0x%x, 0x%x, 0x%x\n", - roce_read(hr_dev, ROCEE_ECC_UCERR_ALM0_REG), - roce_read(hr_dev, ROCEE_ECC_UCERR_ALM1_REG), - roce_read(hr_dev, ROCEE_ECC_UCERR_ALM2_REG)); - - dev_warn(dev, "ECC CERR ALARM: 0x%x, 0x%x, 0x%x\n", - roce_read(hr_dev, ROCEE_ECC_CERR_ALM0_REG), - roce_read(hr_dev, ROCEE_ECC_CERR_ALM1_REG), - roce_read(hr_dev, ROCEE_ECC_CERR_ALM2_REG)); - - return eqovf_found; -} - -static int hns_roce_eq_int(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) -{ - int eqes_found = 0; - - if (likely(eq->type_flag == HNS_ROCE_CEQ)) - /* CEQ irq routine, CEQ is pulse irq, not clear */ - eqes_found = hns_roce_ceq_int(hr_dev, eq); - else if (likely(eq->type_flag == HNS_ROCE_AEQ)) - /* AEQ irq routine, AEQ is pulse irq, not clear */ - eqes_found = hns_roce_aeq_int(hr_dev, eq); - else - /* AEQ queue overflow irq */ - eqes_found = hns_roce_aeq_ovf_int(hr_dev, eq); - - return eqes_found; -} - -static irqreturn_t hns_roce_msi_x_interrupt(int irq, void *eq_ptr) -{ - int int_work = 0; - struct hns_roce_eq *eq = eq_ptr; - struct hns_roce_dev *hr_dev = eq->hr_dev; - - int_work = hns_roce_eq_int(hr_dev, eq); - - return IRQ_RETVAL(int_work); -} - -static void hns_roce_enable_eq(struct hns_roce_dev *hr_dev, int eq_num, - int enable_flag) -{ - void __iomem *eqc = hr_dev->eq_table.eqc_base[eq_num]; - u32 val; - - val = readl(eqc); - - if (enable_flag) - roce_set_field(val, - ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M, - ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S, - HNS_ROCE_EQ_STAT_VALID); - else - roce_set_field(val, - ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M, - ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S, - HNS_ROCE_EQ_STAT_INVALID); - writel(val, eqc); -} - -static int hns_roce_create_eq(struct hns_roce_dev *hr_dev, - struct hns_roce_eq *eq) -{ - void __iomem *eqc = hr_dev->eq_table.eqc_base[eq->eqn]; - struct device *dev = &hr_dev->pdev->dev; - dma_addr_t tmp_dma_addr; - u32 eqconsindx_val = 0; - u32 eqcuridx_val = 0; - u32 eqshift_val = 0; - int num_bas = 0; - int ret; - int i; - - num_bas = (PAGE_ALIGN(eq->entries * eq->eqe_size) + - HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE; - - if ((eq->entries * eq->eqe_size) > HNS_ROCE_BA_SIZE) { - dev_err(dev, "[error]eq buf %d gt ba size(%d) need bas=%d\n", - (eq->entries * eq->eqe_size), HNS_ROCE_BA_SIZE, - num_bas); - return -EINVAL; - } - - eq->buf_list = kcalloc(num_bas, sizeof(*eq->buf_list), GFP_KERNEL); - if (!eq->buf_list) - return -ENOMEM; - - for (i = 0; i < num_bas; ++i) { - eq->buf_list[i].buf = dma_alloc_coherent(dev, HNS_ROCE_BA_SIZE, - &tmp_dma_addr, - GFP_KERNEL); - if (!eq->buf_list[i].buf) { - ret = -ENOMEM; - goto err_out_free_pages; - } - - eq->buf_list[i].map = tmp_dma_addr; - memset(eq->buf_list[i].buf, 0, HNS_ROCE_BA_SIZE); - } - eq->cons_index = 0; - roce_set_field(eqshift_val, - ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M, - ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S, - HNS_ROCE_EQ_STAT_INVALID); - roce_set_field(eqshift_val, - ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_M, - ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S, - eq->log_entries); - writel(eqshift_val, eqc); - - /* Configure eq extended address 12~44bit */ - writel((u32)(eq->buf_list[0].map >> 12), eqc + 4); - - /* - * Configure eq extended address 45~49 bit. - * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of - * using 4K page, and shift more 32 because of - * caculating the high 32 bit value evaluated to hardware. - */ - roce_set_field(eqcuridx_val, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M, - ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S, - eq->buf_list[0].map >> 44); - roce_set_field(eqcuridx_val, - ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M, - ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S, 0); - writel(eqcuridx_val, eqc + 8); - - /* Configure eq consumer index */ - roce_set_field(eqconsindx_val, - ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M, - ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S, 0); - writel(eqconsindx_val, eqc + 0xc); - - return 0; - -err_out_free_pages: - for (i = i - 1; i >= 0; i--) - dma_free_coherent(dev, HNS_ROCE_BA_SIZE, eq->buf_list[i].buf, - eq->buf_list[i].map); - - kfree(eq->buf_list); - return ret; -} - -static void hns_roce_free_eq(struct hns_roce_dev *hr_dev, - struct hns_roce_eq *eq) -{ - int i = 0; - int npages = (PAGE_ALIGN(eq->eqe_size * eq->entries) + - HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE; - - if (!eq->buf_list) - return; - - for (i = 0; i < npages; ++i) - dma_free_coherent(&hr_dev->pdev->dev, HNS_ROCE_BA_SIZE, - eq->buf_list[i].buf, eq->buf_list[i].map); - - kfree(eq->buf_list); -} - -static void hns_roce_int_mask_en(struct hns_roce_dev *hr_dev) -{ - int i = 0; - u32 aemask_val; - int masken = 0; - - /* AEQ INT */ - aemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG); - roce_set_bit(aemask_val, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S, - masken); - roce_set_bit(aemask_val, ROCEE_CAEP_AE_MASK_CAEP_AE_IRQ_MASK_S, masken); - roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, aemask_val); - - /* CEQ INT */ - for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) { - /* IRQ mask */ - roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG + - i * CEQ_REG_OFFSET, masken); - } -} - -static void hns_roce_ce_int_default_cfg(struct hns_roce_dev *hr_dev) -{ - /* Configure ce int interval */ - roce_write(hr_dev, ROCEE_CAEP_CE_INTERVAL_CFG_REG, - HNS_ROCE_CEQ_DEFAULT_INTERVAL); - - /* Configure ce int burst num */ - roce_write(hr_dev, ROCEE_CAEP_CE_BURST_NUM_CFG_REG, - HNS_ROCE_CEQ_DEFAULT_BURST_NUM); -} - -int hns_roce_init_eq_table(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; - struct device *dev = &hr_dev->pdev->dev; - struct hns_roce_eq *eq = NULL; - int eq_num = 0; - int ret = 0; - int i = 0; - int j = 0; - - eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; - eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL); - if (!eq_table->eq) - return -ENOMEM; - - eq_table->eqc_base = kcalloc(eq_num, sizeof(*eq_table->eqc_base), - GFP_KERNEL); - if (!eq_table->eqc_base) { - ret = -ENOMEM; - goto err_eqc_base_alloc_fail; - } - - for (i = 0; i < eq_num; i++) { - eq = &eq_table->eq[i]; - eq->hr_dev = hr_dev; - eq->eqn = i; - eq->irq = hr_dev->irq[i]; - eq->log_page_size = PAGE_SHIFT; - - if (i < hr_dev->caps.num_comp_vectors) { - /* CEQ */ - eq_table->eqc_base[i] = hr_dev->reg_base + - ROCEE_CAEP_CEQC_SHIFT_0_REG + - HNS_ROCE_CEQC_REG_OFFSET * i; - eq->type_flag = HNS_ROCE_CEQ; - eq->doorbell = hr_dev->reg_base + - ROCEE_CAEP_CEQC_CONS_IDX_0_REG + - HNS_ROCE_CEQC_REG_OFFSET * i; - eq->entries = hr_dev->caps.ceqe_depth[i]; - eq->log_entries = ilog2(eq->entries); - eq->eqe_size = sizeof(struct hns_roce_ceqe); - } else { - /* AEQ */ - eq_table->eqc_base[i] = hr_dev->reg_base + - ROCEE_CAEP_AEQC_AEQE_SHIFT_REG; - eq->type_flag = HNS_ROCE_AEQ; - eq->doorbell = hr_dev->reg_base + - ROCEE_CAEP_AEQE_CONS_IDX_REG; - eq->entries = hr_dev->caps.aeqe_depth; - eq->log_entries = ilog2(eq->entries); - eq->eqe_size = sizeof(struct hns_roce_aeqe); - } - } - - /* Disable irq */ - hns_roce_int_mask_en(hr_dev); - - /* Configure CE irq interval and burst num */ - hns_roce_ce_int_default_cfg(hr_dev); - - for (i = 0; i < eq_num; i++) { - ret = hns_roce_create_eq(hr_dev, &eq_table->eq[i]); - if (ret) { - dev_err(dev, "eq create failed\n"); - goto err_create_eq_fail; - } - } - - for (j = 0; j < eq_num; j++) { - ret = request_irq(eq_table->eq[j].irq, hns_roce_msi_x_interrupt, - 0, hr_dev->irq_names[j], eq_table->eq + j); - if (ret) { - dev_err(dev, "request irq error!\n"); - goto err_request_irq_fail; - } - } - - for (i = 0; i < eq_num; i++) - hns_roce_enable_eq(hr_dev, i, EQ_ENABLE); - - return 0; - -err_request_irq_fail: - for (j = j - 1; j >= 0; j--) - free_irq(eq_table->eq[j].irq, eq_table->eq + j); - -err_create_eq_fail: - for (i = i - 1; i >= 0; i--) - hns_roce_free_eq(hr_dev, &eq_table->eq[i]); - - kfree(eq_table->eqc_base); - -err_eqc_base_alloc_fail: - kfree(eq_table->eq); - - return ret; -} - -void hns_roce_cleanup_eq_table(struct hns_roce_dev *hr_dev) -{ - int i; - int eq_num; - struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; - - eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; - for (i = 0; i < eq_num; i++) { - /* Disable EQ */ - hns_roce_enable_eq(hr_dev, i, EQ_DISABLE); - - free_irq(eq_table->eq[i].irq, eq_table->eq + i); - - hns_roce_free_eq(hr_dev, &eq_table->eq[i]); - } - - kfree(eq_table->eqc_base); - kfree(eq_table->eq); -} diff --git a/drivers/infiniband/hw/hns/hns_roce_eq.h b/drivers/infiniband/hw/hns/hns_roce_eq.h deleted file mode 100644 index c6d212d12e03..000000000000 --- a/drivers/infiniband/hw/hns/hns_roce_eq.h +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright (c) 2016 Hisilicon Limited. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef _HNS_ROCE_EQ_H -#define _HNS_ROCE_EQ_H - -#define HNS_ROCE_CEQ 1 -#define HNS_ROCE_AEQ 2 - -#define HNS_ROCE_CEQ_ENTRY_SIZE 0x4 -#define HNS_ROCE_AEQ_ENTRY_SIZE 0x10 -#define HNS_ROCE_CEQC_REG_OFFSET 0x18 - -#define HNS_ROCE_CEQ_DEFAULT_INTERVAL 0x10 -#define HNS_ROCE_CEQ_DEFAULT_BURST_NUM 0x10 - -#define HNS_ROCE_INT_MASK_DISABLE 0 -#define HNS_ROCE_INT_MASK_ENABLE 1 - -#define EQ_ENABLE 1 -#define EQ_DISABLE 0 -#define CONS_INDEX_MASK 0xffff - -#define CEQ_REG_OFFSET 0x18 - -enum { - HNS_ROCE_EQ_STAT_INVALID = 0, - HNS_ROCE_EQ_STAT_VALID = 2, -}; - -struct hns_roce_aeqe { - u32 asyn; - union { - struct { - u32 qp; - u32 rsv0; - u32 rsv1; - } qp_event; - - struct { - u32 cq; - u32 rsv0; - u32 rsv1; - } cq_event; - - struct { - u32 port; - u32 rsv0; - u32 rsv1; - } port_event; - - struct { - u32 ceqe; - u32 rsv0; - u32 rsv1; - } ce_event; - - struct { - __le64 out_param; - __le16 token; - u8 status; - u8 rsv0; - } __packed cmd; - } event; -}; - -#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S 16 -#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M \ - (((1UL << 8) - 1) << HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S) - -#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S 24 -#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M \ - (((1UL << 7) - 1) << HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S) - -#define HNS_ROCE_AEQE_U32_4_OWNER_S 31 - -#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S 0 -#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M \ - (((1UL << 24) - 1) << HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S) - -#define HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S 25 -#define HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M \ - (((1UL << 3) - 1) << HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S) - -#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S 0 -#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M \ - (((1UL << 16) - 1) << HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S) - -#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S 0 -#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M \ - (((1UL << 5) - 1) << HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S) - -struct hns_roce_ceqe { - union { - int comp; - } ceqe; -}; - -#define HNS_ROCE_CEQE_CEQE_COMP_OWNER_S 0 - -#define HNS_ROCE_CEQE_CEQE_COMP_CQN_S 16 -#define HNS_ROCE_CEQE_CEQE_COMP_CQN_M \ - (((1UL << 16) - 1) << HNS_ROCE_CEQE_CEQE_COMP_CQN_S) - -#endif /* _HNS_ROCE_EQ_H */ diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index af27168faf0f..6100ace9f4b6 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -33,6 +33,7 @@ #include <linux/platform_device.h> #include <linux/acpi.h> #include <linux/etherdevice.h> +#include <linux/interrupt.h> #include <linux/of.h> #include <linux/of_platform.h> #include <rdma/ib_umem.h> @@ -1492,9 +1493,9 @@ static int hns_roce_v1_profile(struct hns_roce_dev *hr_dev) caps->max_sq_inline = HNS_ROCE_V1_INLINE_SIZE; caps->num_uars = HNS_ROCE_V1_UAR_NUM; caps->phy_num_uars = HNS_ROCE_V1_PHY_UAR_NUM; - caps->num_aeq_vectors = HNS_ROCE_AEQE_VEC_NUM; - caps->num_comp_vectors = HNS_ROCE_COMP_VEC_NUM; - caps->num_other_vectors = HNS_ROCE_AEQE_OF_VEC_NUM; + caps->num_aeq_vectors = HNS_ROCE_V1_AEQE_VEC_NUM; + caps->num_comp_vectors = HNS_ROCE_V1_COMP_VEC_NUM; + caps->num_other_vectors = HNS_ROCE_V1_ABNORMAL_VEC_NUM; caps->num_mtpts = HNS_ROCE_V1_MAX_MTPT_NUM; caps->num_mtt_segs = HNS_ROCE_V1_MAX_MTT_SEGS; caps->num_pds = HNS_ROCE_V1_MAX_PD_NUM; @@ -1529,10 +1530,8 @@ static int hns_roce_v1_profile(struct hns_roce_dev *hr_dev) caps->num_ports + 1; } - for (i = 0; i < caps->num_comp_vectors; i++) - caps->ceqe_depth[i] = HNS_ROCE_V1_NUM_COMP_EQE; - - caps->aeqe_depth = HNS_ROCE_V1_NUM_ASYNC_EQE; + caps->ceqe_depth = HNS_ROCE_V1_COMP_EQE_NUM; + caps->aeqe_depth = HNS_ROCE_V1_ASYNC_EQE_NUM; caps->local_ca_ack_delay = le32_to_cpu(roce_read(hr_dev, ROCEE_ACK_DELAY_REG)); caps->max_mtu = IB_MTU_2048; @@ -3960,6 +3959,727 @@ static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq) return ret; } +static void set_eq_cons_index_v1(struct hns_roce_eq *eq, int req_not) +{ + roce_raw_write((eq->cons_index & HNS_ROCE_V1_CONS_IDX_M) | + (req_not << eq->log_entries), eq->doorbell); + /* Memory barrier */ + mb(); +} + +static void hns_roce_v1_wq_catas_err_handle(struct hns_roce_dev *hr_dev, + struct hns_roce_aeqe *aeqe, int qpn) +{ + struct device *dev = &hr_dev->pdev->dev; + + dev_warn(dev, "Local Work Queue Catastrophic Error.\n"); + switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M, + HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) { + case HNS_ROCE_LWQCE_QPC_ERROR: + dev_warn(dev, "QP %d, QPC error.\n", qpn); + break; + case HNS_ROCE_LWQCE_MTU_ERROR: + dev_warn(dev, "QP %d, MTU error.\n", qpn); + break; + case HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR: + dev_warn(dev, "QP %d, WQE BA addr error.\n", qpn); + break; + case HNS_ROCE_LWQCE_WQE_ADDR_ERROR: + dev_warn(dev, "QP %d, WQE addr error.\n", qpn); + break; + case HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR: + dev_warn(dev, "QP %d, WQE shift error\n", qpn); + break; + case HNS_ROCE_LWQCE_SL_ERROR: + dev_warn(dev, "QP %d, SL error.\n", qpn); + break; + case HNS_ROCE_LWQCE_PORT_ERROR: + dev_warn(dev, "QP %d, port error.\n", qpn); + break; + default: + break; + } +} + +static void hns_roce_v1_local_wq_access_err_handle(struct hns_roce_dev *hr_dev, + struct hns_roce_aeqe *aeqe, + int qpn) +{ + struct device *dev = &hr_dev->pdev->dev; + + dev_warn(dev, "Local Access Violation Work Queue Error.\n"); + switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M, + HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) { + case HNS_ROCE_LAVWQE_R_KEY_VIOLATION: + dev_warn(dev, "QP %d, R_key violation.\n", qpn); + break; + case HNS_ROCE_LAVWQE_LENGTH_ERROR: + dev_warn(dev, "QP %d, length error.\n", qpn); + break; + case HNS_ROCE_LAVWQE_VA_ERROR: + dev_warn(dev, "QP %d, VA error.\n", qpn); + break; + case HNS_ROCE_LAVWQE_PD_ERROR: + dev_err(dev, "QP %d, PD error.\n", qpn); + break; + case HNS_ROCE_LAVWQE_RW_ACC_ERROR: + dev_warn(dev, "QP %d, rw acc error.\n", qpn); + break; + case HNS_ROCE_LAVWQE_KEY_STATE_ERROR: + dev_warn(dev, "QP %d, key state error.\n", qpn); + break; + case HNS_ROCE_LAVWQE_MR_OPERATION_ERROR: + dev_warn(dev, "QP %d, MR operation error.\n", qpn); + break; + default: + break; + } +} + +static void hns_roce_v1_qp_err_handle(struct hns_roce_dev *hr_dev, + struct hns_roce_aeqe *aeqe, + int event_type) +{ + struct device *dev = &hr_dev->pdev->dev; + int phy_port; + int qpn; + + qpn = roce_get_field(aeqe->event.qp_event.qp, + HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M, + HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S); + phy_port = roce_get_field(aeqe->event.qp_event.qp, + HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M, + HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S); + if (qpn <= 1) + qpn = HNS_ROCE_MAX_PORTS * qpn + phy_port; + + switch (event_type) { + case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: + dev_warn(dev, "Invalid Req Local Work Queue Error.\n" + "QP %d, phy_port %d.\n", qpn, phy_port); + break; + case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + hns_roce_v1_wq_catas_err_handle(hr_dev, aeqe, qpn); + break; + case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: + hns_roce_v1_local_wq_access_err_handle(hr_dev, aeqe, qpn); + break; + default: + break; + } + + hns_roce_qp_event(hr_dev, qpn, event_type); +} + +static void hns_roce_v1_cq_err_handle(struct hns_roce_dev *hr_dev, + struct hns_roce_aeqe *aeqe, + int event_type) +{ + struct device *dev = &hr_dev->pdev->dev; + u32 cqn; + + cqn = le32_to_cpu(roce_get_field(aeqe->event.cq_event.cq, + HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M, + HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S)); + + switch (event_type) { + case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: + dev_warn(dev, "CQ 0x%x access err.\n", cqn); + break; + case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: + dev_warn(dev, "CQ 0x%x overflow\n", cqn); + break; + case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID: + dev_warn(dev, "CQ 0x%x ID invalid.\n", cqn); + break; + default: + break; + } + + hns_roce_cq_event(hr_dev, cqn, event_type); +} + +static void hns_roce_v1_db_overflow_handle(struct hns_roce_dev *hr_dev, + struct hns_roce_aeqe *aeqe) +{ + struct device *dev = &hr_dev->pdev->dev; + + switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M, + HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) { + case HNS_ROCE_DB_SUBTYPE_SDB_OVF: + dev_warn(dev, "SDB overflow.\n"); + break; + case HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF: + dev_warn(dev, "SDB almost overflow.\n"); + break; + case HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP: + dev_warn(dev, "SDB almost empty.\n"); + break; + case HNS_ROCE_DB_SUBTYPE_ODB_OVF: + dev_warn(dev, "ODB overflow.\n"); + break; + case HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF: + dev_warn(dev, "ODB almost overflow.\n"); + break; + case HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP: + dev_warn(dev, "SDB almost empty.\n"); + break; + default: + break; + } +} + +static struct hns_roce_aeqe *get_aeqe_v1(struct hns_roce_eq *eq, u32 entry) +{ + unsigned long off = (entry & (eq->entries - 1)) * + HNS_ROCE_AEQ_ENTRY_SIZE; + + return (struct hns_roce_aeqe *)((u8 *) + (eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) + + off % HNS_ROCE_BA_SIZE); +} + +static struct hns_roce_aeqe *next_aeqe_sw_v1(struct hns_roce_eq *eq) +{ + struct hns_roce_aeqe *aeqe = get_aeqe_v1(eq, eq->cons_index); + + return (roce_get_bit(aeqe->asyn, HNS_ROCE_AEQE_U32_4_OWNER_S) ^ + !!(eq->cons_index & eq->entries)) ? aeqe : NULL; +} + +static int hns_roce_v1_aeq_int(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq) +{ + struct device *dev = &hr_dev->pdev->dev; + struct hns_roce_aeqe *aeqe; + int aeqes_found = 0; + int event_type; + + while ((aeqe = next_aeqe_sw_v1(eq))) { + dev_dbg(dev, "aeqe = %p, aeqe->asyn.event_type = 0x%lx\n", aeqe, + roce_get_field(aeqe->asyn, + HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M, + HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S)); + /* Memory barrier */ + rmb(); + + event_type = roce_get_field(aeqe->asyn, + HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M, + HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S); + switch (event_type) { + case HNS_ROCE_EVENT_TYPE_PATH_MIG: + dev_warn(dev, "PATH MIG not supported\n"); + break; + case HNS_ROCE_EVENT_TYPE_COMM_EST: + dev_warn(dev, "COMMUNICATION established\n"); + break; + case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: + dev_warn(dev, "SQ DRAINED not supported\n"); + break; + case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: + dev_warn(dev, "PATH MIG failed\n"); + break; + case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: + case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: + hns_roce_v1_qp_err_handle(hr_dev, aeqe, event_type); + break; + case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: + case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: + case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: + dev_warn(dev, "SRQ not support!\n"); + break; + case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: + case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: + case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID: + hns_roce_v1_cq_err_handle(hr_dev, aeqe, event_type); + break; + case HNS_ROCE_EVENT_TYPE_PORT_CHANGE: + dev_warn(dev, "port change.\n"); + break; + case HNS_ROCE_EVENT_TYPE_MB: + hns_roce_cmd_event(hr_dev, + le16_to_cpu(aeqe->event.cmd.token), + aeqe->event.cmd.status, + le64_to_cpu(aeqe->event.cmd.out_param + )); + break; + case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW: + hns_roce_v1_db_overflow_handle(hr_dev, aeqe); + break; + case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW: + dev_warn(dev, "CEQ 0x%lx overflow.\n", + roce_get_field(aeqe->event.ce_event.ceqe, + HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M, + HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S)); + break; + default: + dev_warn(dev, "Unhandled event %d on EQ %d at idx %u.\n", + event_type, eq->eqn, eq->cons_index); + break; + } + + eq->cons_index++; + aeqes_found = 1; + + if (eq->cons_index > 2 * hr_dev->caps.aeqe_depth - 1) { + dev_warn(dev, "cons_index overflow, set back to 0.\n"); + eq->cons_index = 0; + } + } + + set_eq_cons_index_v1(eq, 0); + + return aeqes_found; +} + +static struct hns_roce_ceqe *get_ceqe_v1(struct hns_roce_eq *eq, u32 entry) +{ + unsigned long off = (entry & (eq->entries - 1)) * + HNS_ROCE_CEQ_ENTRY_SIZE; + + return (struct hns_roce_ceqe *)((u8 *) + (eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) + + off % HNS_ROCE_BA_SIZE); +} + +static struct hns_roce_ceqe *next_ceqe_sw_v1(struct hns_roce_eq *eq) +{ + struct hns_roce_ceqe *ceqe = get_ceqe_v1(eq, eq->cons_index); + + return (!!(roce_get_bit(ceqe->comp, + HNS_ROCE_CEQE_CEQE_COMP_OWNER_S))) ^ + (!!(eq->cons_index & eq->entries)) ? ceqe : NULL; +} + +static int hns_roce_v1_ceq_int(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq) +{ + struct hns_roce_ceqe *ceqe; + int ceqes_found = 0; + u32 cqn; + + while ((ceqe = next_ceqe_sw_v1(eq))) { + /* Memory barrier */ + rmb(); + cqn = roce_get_field(ceqe->comp, + HNS_ROCE_CEQE_CEQE_COMP_CQN_M, + HNS_ROCE_CEQE_CEQE_COMP_CQN_S); + hns_roce_cq_completion(hr_dev, cqn); + + ++eq->cons_index; + ceqes_found = 1; + + if (eq->cons_index > 2 * hr_dev->caps.ceqe_depth - 1) { + dev_warn(&eq->hr_dev->pdev->dev, + "cons_index overflow, set back to 0.\n"); + eq->cons_index = 0; + } + } + + set_eq_cons_index_v1(eq, 0); + + return ceqes_found; +} + +static irqreturn_t hns_roce_v1_msix_interrupt_eq(int irq, void *eq_ptr) +{ + struct hns_roce_eq *eq = eq_ptr; + struct hns_roce_dev *hr_dev = eq->hr_dev; + int int_work = 0; + + if (eq->type_flag == HNS_ROCE_CEQ) + /* CEQ irq routine, CEQ is pulse irq, not clear */ + int_work = hns_roce_v1_ceq_int(hr_dev, eq); + else + /* AEQ irq routine, AEQ is pulse irq, not clear */ + int_work = hns_roce_v1_aeq_int(hr_dev, eq); + + return IRQ_RETVAL(int_work); +} + +static irqreturn_t hns_roce_v1_msix_interrupt_abn(int irq, void *dev_id) +{ + struct hns_roce_dev *hr_dev = dev_id; + struct device *dev = &hr_dev->pdev->dev; + int int_work = 0; + u32 caepaemask_val; + u32 cealmovf_val; + u32 caepaest_val; + u32 aeshift_val; + u32 ceshift_val; + u32 cemask_val; + int i; + + /* + * Abnormal interrupt: + * AEQ overflow, ECC multi-bit err, CEQ overflow must clear + * interrupt, mask irq, clear irq, cancel mask operation + */ + aeshift_val = roce_read(hr_dev, ROCEE_CAEP_AEQC_AEQE_SHIFT_REG); + + /* AEQE overflow */ + if (roce_get_bit(aeshift_val, + ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQ_ALM_OVF_INT_ST_S) == 1) { + dev_warn(dev, "AEQ overflow!\n"); + + /* Set mask */ + caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG); + roce_set_bit(caepaemask_val, + ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S, + HNS_ROCE_INT_MASK_ENABLE); + roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val); + + /* Clear int state(INT_WC : write 1 clear) */ + caepaest_val = roce_read(hr_dev, ROCEE_CAEP_AE_ST_REG); + roce_set_bit(caepaest_val, + ROCEE_CAEP_AE_ST_CAEP_AEQ_ALM_OVF_S, 1); + roce_write(hr_dev, ROCEE_CAEP_AE_ST_REG, caepaest_val); + + /* Clear mask */ + caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG); + roce_set_bit(caepaemask_val, + ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S, + HNS_ROCE_INT_MASK_DISABLE); + roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val); + } + + /* CEQ almost overflow */ + for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) { + ceshift_val = roce_read(hr_dev, ROCEE_CAEP_CEQC_SHIFT_0_REG + + i * CEQ_REG_OFFSET); + + if (roce_get_bit(ceshift_val, + ROCEE_CAEP_CEQC_SHIFT_CAEP_CEQ_ALM_OVF_INT_ST_S) == 1) { + dev_warn(dev, "CEQ[%d] almost overflow!\n", i); + int_work++; + + /* Set mask */ + cemask_val = roce_read(hr_dev, + ROCEE_CAEP_CE_IRQ_MASK_0_REG + + i * CEQ_REG_OFFSET); + roce_set_bit(cemask_val, + ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S, + HNS_ROCE_INT_MASK_ENABLE); + roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG + + i * CEQ_REG_OFFSET, cemask_val); + + /* Clear int state(INT_WC : write 1 clear) */ + cealmovf_val = roce_read(hr_dev, + ROCEE_CAEP_CEQ_ALM_OVF_0_REG + + i * CEQ_REG_OFFSET); + roce_set_bit(cealmovf_val, + ROCEE_CAEP_CEQ_ALM_OVF_CAEP_CEQ_ALM_OVF_S, + 1); + roce_write(hr_dev, ROCEE_CAEP_CEQ_ALM_OVF_0_REG + + i * CEQ_REG_OFFSET, cealmovf_val); + + /* Clear mask */ + cemask_val = roce_read(hr_dev, + ROCEE_CAEP_CE_IRQ_MASK_0_REG + + i * CEQ_REG_OFFSET); + roce_set_bit(cemask_val, + ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S, + HNS_ROCE_INT_MASK_DISABLE); + roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG + + i * CEQ_REG_OFFSET, cemask_val); + } + } + + /* ECC multi-bit error alarm */ + dev_warn(dev, "ECC UCERR ALARM: 0x%x, 0x%x, 0x%x\n", + roce_read(hr_dev, ROCEE_ECC_UCERR_ALM0_REG), + roce_read(hr_dev, ROCEE_ECC_UCERR_ALM1_REG), + roce_read(hr_dev, ROCEE_ECC_UCERR_ALM2_REG)); + + dev_warn(dev, "ECC CERR ALARM: 0x%x, 0x%x, 0x%x\n", + roce_read(hr_dev, ROCEE_ECC_CERR_ALM0_REG), + roce_read(hr_dev, ROCEE_ECC_CERR_ALM1_REG), + roce_read(hr_dev, ROCEE_ECC_CERR_ALM2_REG)); + + return IRQ_RETVAL(int_work); +} + +static void hns_roce_v1_int_mask_enable(struct hns_roce_dev *hr_dev) +{ + u32 aemask_val; + int masken = 0; + int i; + + /* AEQ INT */ + aemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG); + roce_set_bit(aemask_val, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S, + masken); + roce_set_bit(aemask_val, ROCEE_CAEP_AE_MASK_CAEP_AE_IRQ_MASK_S, masken); + roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, aemask_val); + + /* CEQ INT */ + for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) { + /* IRQ mask */ + roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG + + i * CEQ_REG_OFFSET, masken); + } +} + +static void hns_roce_v1_free_eq(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq) +{ + int npages = (PAGE_ALIGN(eq->eqe_size * eq->entries) + + HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE; + int i; + + if (!eq->buf_list) + return; + + for (i = 0; i < npages; ++i) + dma_free_coherent(&hr_dev->pdev->dev, HNS_ROCE_BA_SIZE, + eq->buf_list[i].buf, eq->buf_list[i].map); + + kfree(eq->buf_list); +} + +static void hns_roce_v1_enable_eq(struct hns_roce_dev *hr_dev, int eq_num, + int enable_flag) +{ + void __iomem *eqc = hr_dev->eq_table.eqc_base[eq_num]; + u32 val; + + val = readl(eqc); + + if (enable_flag) + roce_set_field(val, + ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M, + ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S, + HNS_ROCE_EQ_STAT_VALID); + else + roce_set_field(val, + ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M, + ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S, + HNS_ROCE_EQ_STAT_INVALID); + writel(val, eqc); +} + +static int hns_roce_v1_create_eq(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq) +{ + void __iomem *eqc = hr_dev->eq_table.eqc_base[eq->eqn]; + struct device *dev = &hr_dev->pdev->dev; + dma_addr_t tmp_dma_addr; + u32 eqconsindx_val = 0; + u32 eqcuridx_val = 0; + u32 eqshift_val = 0; + int num_bas; + int ret; + int i; + + num_bas = (PAGE_ALIGN(eq->entries * eq->eqe_size) + + HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE; + + if ((eq->entries * eq->eqe_size) > HNS_ROCE_BA_SIZE) { + dev_err(dev, "[error]eq buf %d gt ba size(%d) need bas=%d\n", + (eq->entries * eq->eqe_size), HNS_ROCE_BA_SIZE, + num_bas); + return -EINVAL; + } + + eq->buf_list = kcalloc(num_bas, sizeof(*eq->buf_list), GFP_KERNEL); + if (!eq->buf_list) + return -ENOMEM; + + for (i = 0; i < num_bas; ++i) { + eq->buf_list[i].buf = dma_alloc_coherent(dev, HNS_ROCE_BA_SIZE, + &tmp_dma_addr, + GFP_KERNEL); + if (!eq->buf_list[i].buf) { + ret = -ENOMEM; + goto err_out_free_pages; + } + + eq->buf_list[i].map = tmp_dma_addr; + memset(eq->buf_list[i].buf, 0, HNS_ROCE_BA_SIZE); + } + eq->cons_index = 0; + roce_set_field(eqshift_val, + ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M, + ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S, + HNS_ROCE_EQ_STAT_INVALID); + roce_set_field(eqshift_val, + ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_M, + ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S, + eq->log_entries); + writel(eqshift_val, eqc); + + /* Configure eq extended address 12~44bit */ + writel((u32)(eq->buf_list[0].map >> 12), eqc + 4); + + /* + * Configure eq extended address 45~49 bit. + * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of + * using 4K page, and shift more 32 because of + * caculating the high 32 bit value evaluated to hardware. + */ + roce_set_field(eqcuridx_val, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M, + ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S, + eq->buf_list[0].map >> 44); + roce_set_field(eqcuridx_val, + ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M, + ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S, 0); + writel(eqcuridx_val, eqc + 8); + + /* Configure eq consumer index */ + roce_set_field(eqconsindx_val, + ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M, + ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S, 0); + writel(eqconsindx_val, eqc + 0xc); + + return 0; + +err_out_free_pages: + for (i -= 1; i >= 0; i--) + dma_free_coherent(dev, HNS_ROCE_BA_SIZE, eq->buf_list[i].buf, + eq->buf_list[i].map); + + kfree(eq->buf_list); + return ret; +} + +static int hns_roce_v1_init_eq_table(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; + struct device *dev = &hr_dev->pdev->dev; + struct hns_roce_eq *eq; + int irq_num; + int eq_num; + int ret; + int i, j; + + eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; + irq_num = eq_num + hr_dev->caps.num_other_vectors; + + eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL); + if (!eq_table->eq) + return -ENOMEM; + + eq_table->eqc_base = kcalloc(eq_num, sizeof(*eq_table->eqc_base), + GFP_KERNEL); + if (!eq_table->eqc_base) { + ret = -ENOMEM; + goto err_eqc_base_alloc_fail; + } + + for (i = 0; i < eq_num; i++) { + eq = &eq_table->eq[i]; + eq->hr_dev = hr_dev; + eq->eqn = i; + eq->irq = hr_dev->irq[i]; + eq->log_page_size = PAGE_SHIFT; + + if (i < hr_dev->caps.num_comp_vectors) { + /* CEQ */ + eq_table->eqc_base[i] = hr_dev->reg_base + + ROCEE_CAEP_CEQC_SHIFT_0_REG + + CEQ_REG_OFFSET * i; + eq->type_flag = HNS_ROCE_CEQ; + eq->doorbell = hr_dev->reg_base + + ROCEE_CAEP_CEQC_CONS_IDX_0_REG + + CEQ_REG_OFFSET * i; + eq->entries = hr_dev->caps.ceqe_depth; + eq->log_entries = ilog2(eq->entries); + eq->eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE; + } else { + /* AEQ */ + eq_table->eqc_base[i] = hr_dev->reg_base + + ROCEE_CAEP_AEQC_AEQE_SHIFT_REG; + eq->type_flag = HNS_ROCE_AEQ; + eq->doorbell = hr_dev->reg_base + + ROCEE_CAEP_AEQE_CONS_IDX_REG; + eq->entries = hr_dev->caps.aeqe_depth; + eq->log_entries = ilog2(eq->entries); + eq->eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE; + } + } + + /* Disable irq */ + hns_roce_v1_int_mask_enable(hr_dev); + + /* Configure ce int interval */ + roce_write(hr_dev, ROCEE_CAEP_CE_INTERVAL_CFG_REG, + HNS_ROCE_CEQ_DEFAULT_INTERVAL); + + /* Configure ce int burst num */ + roce_write(hr_dev, ROCEE_CAEP_CE_BURST_NUM_CFG_REG, + HNS_ROCE_CEQ_DEFAULT_BURST_NUM); + + for (i = 0; i < eq_num; i++) { + ret = hns_roce_v1_create_eq(hr_dev, &eq_table->eq[i]); + if (ret) { + dev_err(dev, "eq create failed\n"); + goto err_create_eq_fail; + } + } + + for (j = 0; j < irq_num; j++) { + if (j < eq_num) + ret = request_irq(hr_dev->irq[j], + hns_roce_v1_msix_interrupt_eq, 0, + hr_dev->irq_names[j], + &eq_table->eq[j]); + else + ret = request_irq(hr_dev->irq[j], + hns_roce_v1_msix_interrupt_abn, 0, + hr_dev->irq_names[j], hr_dev); + + if (ret) { + dev_err(dev, "request irq error!\n"); + goto err_request_irq_fail; + } + } + + for (i = 0; i < eq_num; i++) + hns_roce_v1_enable_eq(hr_dev, i, EQ_ENABLE); + + return 0; + +err_request_irq_fail: + for (j -= 1; j >= 0; j--) + free_irq(hr_dev->irq[j], &eq_table->eq[j]); + +err_create_eq_fail: + for (i -= 1; i >= 0; i--) + hns_roce_v1_free_eq(hr_dev, &eq_table->eq[i]); + + kfree(eq_table->eqc_base); + +err_eqc_base_alloc_fail: + kfree(eq_table->eq); + + return ret; +} + +static void hns_roce_v1_cleanup_eq_table(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; + int irq_num; + int eq_num; + int i; + + eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; + irq_num = eq_num + hr_dev->caps.num_other_vectors; + for (i = 0; i < eq_num; i++) { + /* Disable EQ */ + hns_roce_v1_enable_eq(hr_dev, i, EQ_DISABLE); + + free_irq(hr_dev->irq[i], &eq_table->eq[i]); + + hns_roce_v1_free_eq(hr_dev, &eq_table->eq[i]); + } + for (i = eq_num; i < irq_num; i++) + free_irq(hr_dev->irq[i], hr_dev); + + kfree(eq_table->eqc_base); + kfree(eq_table->eq); +} + static const struct hns_roce_hw hns_roce_hw_v1 = { .reset = hns_roce_v1_reset, .hw_profile = hns_roce_v1_profile, @@ -3983,6 +4703,8 @@ static const struct hns_roce_hw hns_roce_hw_v1 = { .poll_cq = hns_roce_v1_poll_cq, .dereg_mr = hns_roce_v1_dereg_mr, .destroy_cq = hns_roce_v1_destroy_cq, + .init_eq = hns_roce_v1_init_eq_table, + .cleanup_eq = hns_roce_v1_cleanup_eq_table, }; static const struct of_device_id hns_roce_of_match[] = { @@ -4132,14 +4854,14 @@ static int hns_roce_get_cfg(struct hns_roce_dev *hr_dev) /* read the interrupt names from the DT or ACPI */ ret = device_property_read_string_array(dev, "interrupt-names", hr_dev->irq_names, - HNS_ROCE_MAX_IRQ_NUM); + HNS_ROCE_V1_MAX_IRQ_NUM); if (ret < 0) { dev_err(dev, "couldn't get interrupt names from DT or ACPI!\n"); return ret; } /* fetch the interrupt numbers */ - for (i = 0; i < HNS_ROCE_MAX_IRQ_NUM; i++) { + for (i = 0; i < HNS_ROCE_V1_MAX_IRQ_NUM; i++) { hr_dev->irq[i] = platform_get_irq(hr_dev->pdev, i); if (hr_dev->irq[i] <= 0) { dev_err(dev, "platform get of irq[=%d] failed!\n", i); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h index 21a07ef0afc9..b44ddd239060 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h @@ -60,8 +60,13 @@ #define HNS_ROCE_V1_GID_NUM 16 #define HNS_ROCE_V1_RESV_QP 8 -#define HNS_ROCE_V1_NUM_COMP_EQE 0x8000 -#define HNS_ROCE_V1_NUM_ASYNC_EQE 0x400 +#define HNS_ROCE_V1_MAX_IRQ_NUM 34 +#define HNS_ROCE_V1_COMP_VEC_NUM 32 +#define HNS_ROCE_V1_AEQE_VEC_NUM 1 +#define HNS_ROCE_V1_ABNORMAL_VEC_NUM 1 + +#define HNS_ROCE_V1_COMP_EQE_NUM 0x8000 +#define HNS_ROCE_V1_ASYNC_EQE_NUM 0x400 #define HNS_ROCE_V1_QPC_ENTRY_SIZE 256 #define HNS_ROCE_V1_IRRL_ENTRY_SIZE 8 @@ -159,6 +164,41 @@ #define SDB_INV_CNT_OFFSET 8 #define SDB_ST_CMP_VAL 8 +#define HNS_ROCE_CEQ_DEFAULT_INTERVAL 0x10 +#define HNS_ROCE_CEQ_DEFAULT_BURST_NUM 0x10 + +#define HNS_ROCE_INT_MASK_DISABLE 0 +#define HNS_ROCE_INT_MASK_ENABLE 1 + +#define CEQ_REG_OFFSET 0x18 + +#define HNS_ROCE_CEQE_CEQE_COMP_OWNER_S 0 + +#define HNS_ROCE_V1_CONS_IDX_M GENMASK(15, 0) + +#define HNS_ROCE_CEQE_CEQE_COMP_CQN_S 16 +#define HNS_ROCE_CEQE_CEQE_COMP_CQN_M GENMASK(31, 16) + +#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S 16 +#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M GENMASK(23, 16) + +#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S 24 +#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M GENMASK(30, 24) + +#define HNS_ROCE_AEQE_U32_4_OWNER_S 31 + +#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S 0 +#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M GENMASK(23, 0) + +#define HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S 25 +#define HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M GENMASK(27, 25) + +#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S 0 +#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M GENMASK(15, 0) + +#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S 0 +#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M GENMASK(4, 0) + struct hns_roce_cq_context { u32 cqc_byte_4; u32 cq_bt_l; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 8e18445714a9..4d3e97622f39 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -908,9 +908,9 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE; caps->num_uars = HNS_ROCE_V2_UAR_NUM; caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM; - caps->num_aeq_vectors = 1; - caps->num_comp_vectors = 63; - caps->num_other_vectors = 0; + caps->num_aeq_vectors = HNS_ROCE_V2_AEQE_VEC_NUM; + caps->num_comp_vectors = HNS_ROCE_V2_COMP_VEC_NUM; + caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM; caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM; caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS; caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS; @@ -955,12 +955,17 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) caps->cqe_ba_pg_sz = 0; caps->cqe_buf_pg_sz = 0; caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM; + caps->eqe_ba_pg_sz = 0; + caps->eqe_buf_pg_sz = 0; + caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM; caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE; caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR | HNS_ROCE_CAP_FLAG_ROCE_V1_V2; caps->pkey_table_len[0] = 1; caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM; + caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM; + caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM; caps->local_ca_ack_delay = 0; caps->max_mtu = IB_MTU_4096; @@ -1382,6 +1387,8 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M, V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID); + roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M, + V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE); roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M, V2_CQC_BYTE_4_SHIFT_S, ilog2((unsigned int)nent)); roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M, @@ -1422,6 +1429,15 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M, V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3))); + + roce_set_field(cq_context->byte_56_cqe_period_maxcnt, + V2_CQC_BYTE_56_CQ_MAX_CNT_M, + V2_CQC_BYTE_56_CQ_MAX_CNT_S, + HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM); + roce_set_field(cq_context->byte_56_cqe_period_maxcnt, + V2_CQC_BYTE_56_CQ_PERIOD_M, + V2_CQC_BYTE_56_CQ_PERIOD_S, + HNS_ROCE_V2_CQ_DEFAULT_INTERVAL); } static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, @@ -3162,6 +3178,1152 @@ static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) return ret; } +static void set_eq_cons_index_v2(struct hns_roce_eq *eq) +{ + u32 doorbell[2]; + + doorbell[0] = 0; + doorbell[1] = 0; + + if (eq->type_flag == HNS_ROCE_AEQ) { + roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M, + HNS_ROCE_V2_EQ_DB_CMD_S, + eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ? + HNS_ROCE_EQ_DB_CMD_AEQ : + HNS_ROCE_EQ_DB_CMD_AEQ_ARMED); + } else { + roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_TAG_M, + HNS_ROCE_V2_EQ_DB_TAG_S, eq->eqn); + + roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M, + HNS_ROCE_V2_EQ_DB_CMD_S, + eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ? + HNS_ROCE_EQ_DB_CMD_CEQ : + HNS_ROCE_EQ_DB_CMD_CEQ_ARMED); + } + + roce_set_field(doorbell[1], HNS_ROCE_V2_EQ_DB_PARA_M, + HNS_ROCE_V2_EQ_DB_PARA_S, + (eq->cons_index & HNS_ROCE_V2_CONS_IDX_M)); + + hns_roce_write64_k(doorbell, eq->doorbell); + + /* Memory barrier */ + mb(); + +} + +static void hns_roce_v2_wq_catas_err_handle(struct hns_roce_dev *hr_dev, + struct hns_roce_aeqe *aeqe, + u32 qpn) +{ + struct device *dev = hr_dev->dev; + int sub_type; + + dev_warn(dev, "Local work queue catastrophic error.\n"); + sub_type = roce_get_field(aeqe->asyn, HNS_ROCE_V2_AEQE_SUB_TYPE_M, + HNS_ROCE_V2_AEQE_SUB_TYPE_S); + switch (sub_type) { + case HNS_ROCE_LWQCE_QPC_ERROR: + dev_warn(dev, "QP %d, QPC error.\n", qpn); + break; + case HNS_ROCE_LWQCE_MTU_ERROR: + dev_warn(dev, "QP %d, MTU error.\n", qpn); + break; + case HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR: + dev_warn(dev, "QP %d, WQE BA addr error.\n", qpn); + break; + case HNS_ROCE_LWQCE_WQE_ADDR_ERROR: + dev_warn(dev, "QP %d, WQE addr error.\n", qpn); + break; + case HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR: + dev_warn(dev, "QP %d, WQE shift error.\n", qpn); + break; + default: + dev_err(dev, "Unhandled sub_event type %d.\n", sub_type); + break; + } +} + +static void hns_roce_v2_local_wq_access_err_handle(struct hns_roce_dev *hr_dev, + struct hns_roce_aeqe *aeqe, u32 qpn) +{ + struct device *dev = hr_dev->dev; + int sub_type; + + dev_warn(dev, "Local access violation work queue error.\n"); + sub_type = roce_get_field(aeqe->asyn, HNS_ROCE_V2_AEQE_SUB_TYPE_M, + HNS_ROCE_V2_AEQE_SUB_TYPE_S); + switch (sub_type) { + case HNS_ROCE_LAVWQE_R_KEY_VIOLATION: + dev_warn(dev, "QP %d, R_key violation.\n", qpn); + break; + case HNS_ROCE_LAVWQE_LENGTH_ERROR: + dev_warn(dev, "QP %d, length error.\n", qpn); + break; + case HNS_ROCE_LAVWQE_VA_ERROR: + dev_warn(dev, "QP %d, VA error.\n", qpn); + break; + case HNS_ROCE_LAVWQE_PD_ERROR: + dev_err(dev, "QP %d, PD error.\n", qpn); + break; + case HNS_ROCE_LAVWQE_RW_ACC_ERROR: + dev_warn(dev, "QP %d, rw acc error.\n", qpn); + break; + case HNS_ROCE_LAVWQE_KEY_STATE_ERROR: + dev_warn(dev, "QP %d, key state error.\n", qpn); + break; + case HNS_ROCE_LAVWQE_MR_OPERATION_ERROR: + dev_warn(dev, "QP %d, MR operation error.\n", qpn); + break; + default: + dev_err(dev, "Unhandled sub_event type %d.\n", sub_type); + break; + } +} + +static void hns_roce_v2_qp_err_handle(struct hns_roce_dev *hr_dev, + struct hns_roce_aeqe *aeqe, + int event_type) +{ + struct device *dev = hr_dev->dev; + u32 qpn; + + qpn = roce_get_field(aeqe->event.qp_event.qp, + HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M, + HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S); + + switch (event_type) { + case HNS_ROCE_EVENT_TYPE_COMM_EST: + dev_warn(dev, "Communication established.\n"); + break; + case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: + dev_warn(dev, "Send queue drained.\n"); + break; + case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + hns_roce_v2_wq_catas_err_handle(hr_dev, aeqe, qpn); + break; + case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: + dev_warn(dev, "Invalid request local work queue error.\n"); + break; + case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: + hns_roce_v2_local_wq_access_err_handle(hr_dev, aeqe, qpn); + break; + default: + break; + } + + hns_roce_qp_event(hr_dev, qpn, event_type); +} + +static void hns_roce_v2_cq_err_handle(struct hns_roce_dev *hr_dev, + struct hns_roce_aeqe *aeqe, + int event_type) +{ + struct device *dev = hr_dev->dev; + u32 cqn; + + cqn = roce_get_field(aeqe->event.cq_event.cq, + HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M, + HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S); + + switch (event_type) { + case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: + dev_warn(dev, "CQ 0x%x access err.\n", cqn); + break; + case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: + dev_warn(dev, "CQ 0x%x overflow\n", cqn); + break; + default: + break; + } + + hns_roce_cq_event(hr_dev, cqn, event_type); +} + +static struct hns_roce_aeqe *get_aeqe_v2(struct hns_roce_eq *eq, u32 entry) +{ + u32 buf_chk_sz; + unsigned long off; + + buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT); + off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE; + + return (struct hns_roce_aeqe *)((char *)(eq->buf_list->buf) + + off % buf_chk_sz); +} + +static struct hns_roce_aeqe *mhop_get_aeqe(struct hns_roce_eq *eq, u32 entry) +{ + u32 buf_chk_sz; + unsigned long off; + + buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT); + + off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE; + + if (eq->hop_num == HNS_ROCE_HOP_NUM_0) + return (struct hns_roce_aeqe *)((u8 *)(eq->bt_l0) + + off % buf_chk_sz); + else + return (struct hns_roce_aeqe *)((u8 *) + (eq->buf[off / buf_chk_sz]) + off % buf_chk_sz); +} + +static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq) +{ + struct hns_roce_aeqe *aeqe; + + if (!eq->hop_num) + aeqe = get_aeqe_v2(eq, eq->cons_index); + else + aeqe = mhop_get_aeqe(eq, eq->cons_index); + + return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^ + !!(eq->cons_index & eq->entries)) ? aeqe : NULL; +} + +static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq) +{ + struct device *dev = hr_dev->dev; + struct hns_roce_aeqe *aeqe; + int aeqe_found = 0; + int event_type; + + while ((aeqe = next_aeqe_sw_v2(eq))) { + /* Memory barrier */ + rmb(); + + event_type = roce_get_field(aeqe->asyn, + HNS_ROCE_V2_AEQE_EVENT_TYPE_M, + HNS_ROCE_V2_AEQE_EVENT_TYPE_S); + + switch (event_type) { + case HNS_ROCE_EVENT_TYPE_PATH_MIG: + dev_warn(dev, "Path migrated succeeded.\n"); + break; + case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: + dev_warn(dev, "Path migration failed.\n"); + break; + case HNS_ROCE_EVENT_TYPE_COMM_EST: + case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: + case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: + case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: + hns_roce_v2_qp_err_handle(hr_dev, aeqe, event_type); + break; + case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: + case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: + case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: + dev_warn(dev, "SRQ not support.\n"); + break; + case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: + case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: + hns_roce_v2_cq_err_handle(hr_dev, aeqe, event_type); + break; + case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW: + dev_warn(dev, "DB overflow.\n"); + break; + case HNS_ROCE_EVENT_TYPE_MB: + hns_roce_cmd_event(hr_dev, + le16_to_cpu(aeqe->event.cmd.token), + aeqe->event.cmd.status, + le64_to_cpu(aeqe->event.cmd.out_param)); + break; + case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW: + dev_warn(dev, "CEQ overflow.\n"); + break; + case HNS_ROCE_EVENT_TYPE_FLR: + dev_warn(dev, "Function level reset.\n"); + break; + default: + dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n", + event_type, eq->eqn, eq->cons_index); + break; + }; + + ++eq->cons_index; + aeqe_found = 1; + + if (eq->cons_index > (2 * eq->entries - 1)) { + dev_warn(dev, "cons_index overflow, set back to 0.\n"); + eq->cons_index = 0; + } + } + + set_eq_cons_index_v2(eq); + return aeqe_found; +} + +static struct hns_roce_ceqe *get_ceqe_v2(struct hns_roce_eq *eq, u32 entry) +{ + u32 buf_chk_sz; + unsigned long off; + + buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT); + off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE; + + return (struct hns_roce_ceqe *)((char *)(eq->buf_list->buf) + + off % buf_chk_sz); +} + +static struct hns_roce_ceqe *mhop_get_ceqe(struct hns_roce_eq *eq, u32 entry) +{ + u32 buf_chk_sz; + unsigned long off; + + buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT); + + off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE; + + if (eq->hop_num == HNS_ROCE_HOP_NUM_0) + return (struct hns_roce_ceqe *)((u8 *)(eq->bt_l0) + + off % buf_chk_sz); + else + return (struct hns_roce_ceqe *)((u8 *)(eq->buf[off / + buf_chk_sz]) + off % buf_chk_sz); +} + +static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq) +{ + struct hns_roce_ceqe *ceqe; + + if (!eq->hop_num) + ceqe = get_ceqe_v2(eq, eq->cons_index); + else + ceqe = mhop_get_ceqe(eq, eq->cons_index); + + return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^ + (!!(eq->cons_index & eq->entries)) ? ceqe : NULL; +} + +static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq) +{ + struct device *dev = hr_dev->dev; + struct hns_roce_ceqe *ceqe; + int ceqe_found = 0; + u32 cqn; + + while ((ceqe = next_ceqe_sw_v2(eq))) { + + /* Memory barrier */ + rmb(); + cqn = roce_get_field(ceqe->comp, + HNS_ROCE_V2_CEQE_COMP_CQN_M, + HNS_ROCE_V2_CEQE_COMP_CQN_S); + + hns_roce_cq_completion(hr_dev, cqn); + + ++eq->cons_index; + ceqe_found = 1; + + if (eq->cons_index > (2 * eq->entries - 1)) { + dev_warn(dev, "cons_index overflow, set back to 0.\n"); + eq->cons_index = 0; + } + } + + set_eq_cons_index_v2(eq); + + return ceqe_found; +} + +static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr) +{ + struct hns_roce_eq *eq = eq_ptr; + struct hns_roce_dev *hr_dev = eq->hr_dev; + int int_work = 0; + + if (eq->type_flag == HNS_ROCE_CEQ) + /* Completion event interrupt */ + int_work = hns_roce_v2_ceq_int(hr_dev, eq); + else + /* Asychronous event interrupt */ + int_work = hns_roce_v2_aeq_int(hr_dev, eq); + + return IRQ_RETVAL(int_work); +} + +static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id) +{ + struct hns_roce_dev *hr_dev = dev_id; + struct device *dev = hr_dev->dev; + int int_work = 0; + u32 int_st; + u32 int_en; + + /* Abnormal interrupt */ + int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG); + int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG); + + if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) { + dev_err(dev, "AEQ overflow!\n"); + + roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S, 1); + roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st); + + /* Memory barrier */ + mb(); + + roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1); + roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); + + int_work = 1; + } else if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) { + dev_err(dev, "BUS ERR!\n"); + + roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S, 1); + roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st); + + /* Memory barrier */ + mb(); + + roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1); + roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); + + int_work = 1; + } else if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) { + dev_err(dev, "OTHER ERR!\n"); + + roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S, 1); + roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st); + + /* Memory barrier */ + mb(); + roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1); + roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); + + int_work = 1; + } else + dev_err(dev, "There is no abnormal irq found!\n"); + + return IRQ_RETVAL(int_work); +} + +static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev, + int eq_num, int enable_flag) +{ + int i; + + if (enable_flag == EQ_ENABLE) { + for (i = 0; i < eq_num; i++) + roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG + + i * EQ_REG_OFFSET, + HNS_ROCE_V2_VF_EVENT_INT_EN_M); + + roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, + HNS_ROCE_V2_VF_ABN_INT_EN_M); + roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, + HNS_ROCE_V2_VF_ABN_INT_CFG_M); + } else { + for (i = 0; i < eq_num; i++) + roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG + + i * EQ_REG_OFFSET, + HNS_ROCE_V2_VF_EVENT_INT_EN_M & 0x0); + + roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, + HNS_ROCE_V2_VF_ABN_INT_EN_M & 0x0); + roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, + HNS_ROCE_V2_VF_ABN_INT_CFG_M & 0x0); + } +} + +static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn) +{ + struct device *dev = hr_dev->dev; + int ret; + + if (eqn < hr_dev->caps.num_comp_vectors) + ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M, + 0, HNS_ROCE_CMD_DESTROY_CEQC, + HNS_ROCE_CMD_TIMEOUT_MSECS); + else + ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M, + 0, HNS_ROCE_CMD_DESTROY_AEQC, + HNS_ROCE_CMD_TIMEOUT_MSECS); + if (ret) + dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn); +} + +static void hns_roce_mhop_free_eq(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq) +{ + struct device *dev = hr_dev->dev; + u64 idx; + u64 size; + u32 buf_chk_sz; + u32 bt_chk_sz; + u32 mhop_num; + int eqe_alloc; + int ba_num; + int i = 0; + int j = 0; + + mhop_num = hr_dev->caps.eqe_hop_num; + buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT); + bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT); + ba_num = (PAGE_ALIGN(eq->entries * eq->eqe_size) + buf_chk_sz - 1) / + buf_chk_sz; + + /* hop_num = 0 */ + if (mhop_num == HNS_ROCE_HOP_NUM_0) { + dma_free_coherent(dev, (unsigned int)(eq->entries * + eq->eqe_size), eq->bt_l0, eq->l0_dma); + return; + } + + /* hop_num = 1 or hop = 2 */ + dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma); + if (mhop_num == 1) { + for (i = 0; i < eq->l0_last_num; i++) { + if (i == eq->l0_last_num - 1) { + eqe_alloc = i * (buf_chk_sz / eq->eqe_size); + size = (eq->entries - eqe_alloc) * eq->eqe_size; + dma_free_coherent(dev, size, eq->buf[i], + eq->buf_dma[i]); + break; + } + dma_free_coherent(dev, buf_chk_sz, eq->buf[i], + eq->buf_dma[i]); + } + } else if (mhop_num == 2) { + for (i = 0; i < eq->l0_last_num; i++) { + dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i], + eq->l1_dma[i]); + + for (j = 0; j < bt_chk_sz / 8; j++) { + idx = i * (bt_chk_sz / 8) + j; + if ((i == eq->l0_last_num - 1) + && j == eq->l1_last_num - 1) { + eqe_alloc = (buf_chk_sz / eq->eqe_size) + * idx; + size = (eq->entries - eqe_alloc) + * eq->eqe_size; + dma_free_coherent(dev, size, + eq->buf[idx], + eq->buf_dma[idx]); + break; + } + dma_free_coherent(dev, buf_chk_sz, eq->buf[idx], + eq->buf_dma[idx]); + } + } + } + kfree(eq->buf_dma); + kfree(eq->buf); + kfree(eq->l1_dma); + kfree(eq->bt_l1); + eq->buf_dma = NULL; + eq->buf = NULL; + eq->l1_dma = NULL; + eq->bt_l1 = NULL; +} + +static void hns_roce_v2_free_eq(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq) +{ + u32 buf_chk_sz; + + buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT); + + if (hr_dev->caps.eqe_hop_num) { + hns_roce_mhop_free_eq(hr_dev, eq); + return; + } + + if (eq->buf_list) + dma_free_coherent(hr_dev->dev, buf_chk_sz, + eq->buf_list->buf, eq->buf_list->map); +} + +static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq, + void *mb_buf) +{ + struct hns_roce_eq_context *eqc; + + eqc = mb_buf; + memset(eqc, 0, sizeof(struct hns_roce_eq_context)); + + /* init eqc */ + eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG; + eq->hop_num = hr_dev->caps.eqe_hop_num; + eq->cons_index = 0; + eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0; + eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0; + eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED; + eq->eqe_ba_pg_sz = hr_dev->caps.eqe_ba_pg_sz; + eq->eqe_buf_pg_sz = hr_dev->caps.eqe_buf_pg_sz; + eq->shift = ilog2((unsigned int)eq->entries); + + if (!eq->hop_num) + eq->eqe_ba = eq->buf_list->map; + else + eq->eqe_ba = eq->l0_dma; + + /* set eqc state */ + roce_set_field(eqc->byte_4, + HNS_ROCE_EQC_EQ_ST_M, + HNS_ROCE_EQC_EQ_ST_S, + HNS_ROCE_V2_EQ_STATE_VALID); + + /* set eqe hop num */ + roce_set_field(eqc->byte_4, + HNS_ROCE_EQC_HOP_NUM_M, + HNS_ROCE_EQC_HOP_NUM_S, eq->hop_num); + + /* set eqc over_ignore */ + roce_set_field(eqc->byte_4, + HNS_ROCE_EQC_OVER_IGNORE_M, + HNS_ROCE_EQC_OVER_IGNORE_S, eq->over_ignore); + + /* set eqc coalesce */ + roce_set_field(eqc->byte_4, + HNS_ROCE_EQC_COALESCE_M, + HNS_ROCE_EQC_COALESCE_S, eq->coalesce); + + /* set eqc arm_state */ + roce_set_field(eqc->byte_4, + HNS_ROCE_EQC_ARM_ST_M, + HNS_ROCE_EQC_ARM_ST_S, eq->arm_st); + + /* set eqn */ + roce_set_field(eqc->byte_4, + HNS_ROCE_EQC_EQN_M, + HNS_ROCE_EQC_EQN_S, eq->eqn); + + /* set eqe_cnt */ + roce_set_field(eqc->byte_4, + HNS_ROCE_EQC_EQE_CNT_M, + HNS_ROCE_EQC_EQE_CNT_S, + HNS_ROCE_EQ_INIT_EQE_CNT); + + /* set eqe_ba_pg_sz */ + roce_set_field(eqc->byte_8, + HNS_ROCE_EQC_BA_PG_SZ_M, + HNS_ROCE_EQC_BA_PG_SZ_S, eq->eqe_ba_pg_sz); + + /* set eqe_buf_pg_sz */ + roce_set_field(eqc->byte_8, + HNS_ROCE_EQC_BUF_PG_SZ_M, + HNS_ROCE_EQC_BUF_PG_SZ_S, eq->eqe_buf_pg_sz); + + /* set eq_producer_idx */ + roce_set_field(eqc->byte_8, + HNS_ROCE_EQC_PROD_INDX_M, + HNS_ROCE_EQC_PROD_INDX_S, + HNS_ROCE_EQ_INIT_PROD_IDX); + + /* set eq_max_cnt */ + roce_set_field(eqc->byte_12, + HNS_ROCE_EQC_MAX_CNT_M, + HNS_ROCE_EQC_MAX_CNT_S, eq->eq_max_cnt); + + /* set eq_period */ + roce_set_field(eqc->byte_12, + HNS_ROCE_EQC_PERIOD_M, + HNS_ROCE_EQC_PERIOD_S, eq->eq_period); + + /* set eqe_report_timer */ + roce_set_field(eqc->eqe_report_timer, + HNS_ROCE_EQC_REPORT_TIMER_M, + HNS_ROCE_EQC_REPORT_TIMER_S, + HNS_ROCE_EQ_INIT_REPORT_TIMER); + + /* set eqe_ba [34:3] */ + roce_set_field(eqc->eqe_ba0, + HNS_ROCE_EQC_EQE_BA_L_M, + HNS_ROCE_EQC_EQE_BA_L_S, eq->eqe_ba >> 3); + + /* set eqe_ba [64:35] */ + roce_set_field(eqc->eqe_ba1, + HNS_ROCE_EQC_EQE_BA_H_M, + HNS_ROCE_EQC_EQE_BA_H_S, eq->eqe_ba >> 35); + + /* set eq shift */ + roce_set_field(eqc->byte_28, + HNS_ROCE_EQC_SHIFT_M, + HNS_ROCE_EQC_SHIFT_S, eq->shift); + + /* set eq MSI_IDX */ + roce_set_field(eqc->byte_28, + HNS_ROCE_EQC_MSI_INDX_M, + HNS_ROCE_EQC_MSI_INDX_S, + HNS_ROCE_EQ_INIT_MSI_IDX); + + /* set cur_eqe_ba [27:12] */ + roce_set_field(eqc->byte_28, + HNS_ROCE_EQC_CUR_EQE_BA_L_M, + HNS_ROCE_EQC_CUR_EQE_BA_L_S, eq->cur_eqe_ba >> 12); + + /* set cur_eqe_ba [59:28] */ + roce_set_field(eqc->byte_32, + HNS_ROCE_EQC_CUR_EQE_BA_M_M, + HNS_ROCE_EQC_CUR_EQE_BA_M_S, eq->cur_eqe_ba >> 28); + + /* set cur_eqe_ba [63:60] */ + roce_set_field(eqc->byte_36, + HNS_ROCE_EQC_CUR_EQE_BA_H_M, + HNS_ROCE_EQC_CUR_EQE_BA_H_S, eq->cur_eqe_ba >> 60); + + /* set eq consumer idx */ + roce_set_field(eqc->byte_36, + HNS_ROCE_EQC_CONS_INDX_M, + HNS_ROCE_EQC_CONS_INDX_S, + HNS_ROCE_EQ_INIT_CONS_IDX); + + /* set nex_eqe_ba[43:12] */ + roce_set_field(eqc->nxt_eqe_ba0, + HNS_ROCE_EQC_NXT_EQE_BA_L_M, + HNS_ROCE_EQC_NXT_EQE_BA_L_S, eq->nxt_eqe_ba >> 12); + + /* set nex_eqe_ba[63:44] */ + roce_set_field(eqc->nxt_eqe_ba1, + HNS_ROCE_EQC_NXT_EQE_BA_H_M, + HNS_ROCE_EQC_NXT_EQE_BA_H_S, eq->nxt_eqe_ba >> 44); +} + +static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq) +{ + struct device *dev = hr_dev->dev; + int eq_alloc_done = 0; + int eq_buf_cnt = 0; + int eqe_alloc; + u32 buf_chk_sz; + u32 bt_chk_sz; + u32 mhop_num; + u64 size; + u64 idx; + int ba_num; + int bt_num; + int record_i; + int record_j; + int i = 0; + int j = 0; + + mhop_num = hr_dev->caps.eqe_hop_num; + buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT); + bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT); + + ba_num = (PAGE_ALIGN(eq->entries * eq->eqe_size) + buf_chk_sz - 1) + / buf_chk_sz; + bt_num = (ba_num + bt_chk_sz / 8 - 1) / (bt_chk_sz / 8); + + /* hop_num = 0 */ + if (mhop_num == HNS_ROCE_HOP_NUM_0) { + if (eq->entries > buf_chk_sz / eq->eqe_size) { + dev_err(dev, "eq entries %d is larger than buf_pg_sz!", + eq->entries); + return -EINVAL; + } + eq->bt_l0 = dma_alloc_coherent(dev, eq->entries * eq->eqe_size, + &(eq->l0_dma), GFP_KERNEL); + if (!eq->bt_l0) + return -ENOMEM; + + eq->cur_eqe_ba = eq->l0_dma; + eq->nxt_eqe_ba = 0; + + memset(eq->bt_l0, 0, eq->entries * eq->eqe_size); + + return 0; + } + + eq->buf_dma = kcalloc(ba_num, sizeof(*eq->buf_dma), GFP_KERNEL); + if (!eq->buf_dma) + return -ENOMEM; + eq->buf = kcalloc(ba_num, sizeof(*eq->buf), GFP_KERNEL); + if (!eq->buf) + goto err_kcalloc_buf; + + if (mhop_num == 2) { + eq->l1_dma = kcalloc(bt_num, sizeof(*eq->l1_dma), GFP_KERNEL); + if (!eq->l1_dma) + goto err_kcalloc_l1_dma; + + eq->bt_l1 = kcalloc(bt_num, sizeof(*eq->bt_l1), GFP_KERNEL); + if (!eq->bt_l1) + goto err_kcalloc_bt_l1; + } + + /* alloc L0 BT */ + eq->bt_l0 = dma_alloc_coherent(dev, bt_chk_sz, &eq->l0_dma, GFP_KERNEL); + if (!eq->bt_l0) + goto err_dma_alloc_l0; + + if (mhop_num == 1) { + if (ba_num > (bt_chk_sz / 8)) + dev_err(dev, "ba_num %d is too large for 1 hop\n", + ba_num); + + /* alloc buf */ + for (i = 0; i < bt_chk_sz / 8; i++) { + if (eq_buf_cnt + 1 < ba_num) { + size = buf_chk_sz; + } else { + eqe_alloc = i * (buf_chk_sz / eq->eqe_size); + size = (eq->entries - eqe_alloc) * eq->eqe_size; + } + eq->buf[i] = dma_alloc_coherent(dev, size, + &(eq->buf_dma[i]), + GFP_KERNEL); + if (!eq->buf[i]) + goto err_dma_alloc_buf; + + memset(eq->buf[i], 0, size); + *(eq->bt_l0 + i) = eq->buf_dma[i]; + + eq_buf_cnt++; + if (eq_buf_cnt >= ba_num) + break; + } + eq->cur_eqe_ba = eq->buf_dma[0]; + eq->nxt_eqe_ba = eq->buf_dma[1]; + + } else if (mhop_num == 2) { + /* alloc L1 BT and buf */ + for (i = 0; i < bt_chk_sz / 8; i++) { + eq->bt_l1[i] = dma_alloc_coherent(dev, bt_chk_sz, + &(eq->l1_dma[i]), + GFP_KERNEL); + if (!eq->bt_l1[i]) + goto err_dma_alloc_l1; + *(eq->bt_l0 + i) = eq->l1_dma[i]; + + for (j = 0; j < bt_chk_sz / 8; j++) { + idx = i * bt_chk_sz / 8 + j; + if (eq_buf_cnt + 1 < ba_num) { + size = buf_chk_sz; + } else { + eqe_alloc = (buf_chk_sz / eq->eqe_size) + * idx; + size = (eq->entries - eqe_alloc) + * eq->eqe_size; + } + eq->buf[idx] = dma_alloc_coherent(dev, size, + &(eq->buf_dma[idx]), + GFP_KERNEL); + if (!eq->buf[idx]) + goto err_dma_alloc_buf; + + memset(eq->buf[idx], 0, size); + *(eq->bt_l1[i] + j) = eq->buf_dma[idx]; + + eq_buf_cnt++; + if (eq_buf_cnt >= ba_num) { + eq_alloc_done = 1; + break; + } + } + + if (eq_alloc_done) + break; + } + eq->cur_eqe_ba = eq->buf_dma[0]; + eq->nxt_eqe_ba = eq->buf_dma[1]; + } + + eq->l0_last_num = i + 1; + if (mhop_num == 2) + eq->l1_last_num = j + 1; + + return 0; + +err_dma_alloc_l1: + dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma); + eq->bt_l0 = NULL; + eq->l0_dma = 0; + for (i -= 1; i >= 0; i--) { + dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i], + eq->l1_dma[i]); + + for (j = 0; j < bt_chk_sz / 8; j++) { + idx = i * bt_chk_sz / 8 + j; + dma_free_coherent(dev, buf_chk_sz, eq->buf[idx], + eq->buf_dma[idx]); + } + } + goto err_dma_alloc_l0; + +err_dma_alloc_buf: + dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma); + eq->bt_l0 = NULL; + eq->l0_dma = 0; + + if (mhop_num == 1) + for (i -= i; i >= 0; i--) + dma_free_coherent(dev, buf_chk_sz, eq->buf[i], + eq->buf_dma[i]); + else if (mhop_num == 2) { + record_i = i; + record_j = j; + for (; i >= 0; i--) { + dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i], + eq->l1_dma[i]); + + for (j = 0; j < bt_chk_sz / 8; j++) { + if (i == record_i && j >= record_j) + break; + + idx = i * bt_chk_sz / 8 + j; + dma_free_coherent(dev, buf_chk_sz, + eq->buf[idx], + eq->buf_dma[idx]); + } + } + } + +err_dma_alloc_l0: + kfree(eq->bt_l1); + eq->bt_l1 = NULL; + +err_kcalloc_bt_l1: + kfree(eq->l1_dma); + eq->l1_dma = NULL; + +err_kcalloc_l1_dma: + kfree(eq->buf); + eq->buf = NULL; + +err_kcalloc_buf: + kfree(eq->buf_dma); + eq->buf_dma = NULL; + + return -ENOMEM; +} + +static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq, + unsigned int eq_cmd) +{ + struct device *dev = hr_dev->dev; + struct hns_roce_cmd_mailbox *mailbox; + u32 buf_chk_sz = 0; + int ret; + + /* Allocate mailbox memory */ + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + if (!hr_dev->caps.eqe_hop_num) { + buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT); + + eq->buf_list = kzalloc(sizeof(struct hns_roce_buf_list), + GFP_KERNEL); + if (!eq->buf_list) { + ret = -ENOMEM; + goto free_cmd_mbox; + } + + eq->buf_list->buf = dma_alloc_coherent(dev, buf_chk_sz, + &(eq->buf_list->map), + GFP_KERNEL); + if (!eq->buf_list->buf) { + ret = -ENOMEM; + goto err_alloc_buf; + } + + memset(eq->buf_list->buf, 0, buf_chk_sz); + } else { + ret = hns_roce_mhop_alloc_eq(hr_dev, eq); + if (ret) { + ret = -ENOMEM; + goto free_cmd_mbox; + } + } + + hns_roce_config_eqc(hr_dev, eq, mailbox->buf); + + ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0, + eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS); + if (ret) { + dev_err(dev, "[mailbox cmd] creat eqc failed.\n"); + goto err_cmd_mbox; + } + + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return 0; + +err_cmd_mbox: + if (!hr_dev->caps.eqe_hop_num) + dma_free_coherent(dev, buf_chk_sz, eq->buf_list->buf, + eq->buf_list->map); + else { + hns_roce_mhop_free_eq(hr_dev, eq); + goto free_cmd_mbox; + } + +err_alloc_buf: + kfree(eq->buf_list); + +free_cmd_mbox: + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return ret; +} + +static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; + struct device *dev = hr_dev->dev; + struct hns_roce_eq *eq; + unsigned int eq_cmd; + int irq_num; + int eq_num; + int other_num; + int comp_num; + int aeq_num; + int i, j, k; + int ret; + + other_num = hr_dev->caps.num_other_vectors; + comp_num = hr_dev->caps.num_comp_vectors; + aeq_num = hr_dev->caps.num_aeq_vectors; + + eq_num = comp_num + aeq_num; + irq_num = eq_num + other_num; + + eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL); + if (!eq_table->eq) + return -ENOMEM; + + for (i = 0; i < irq_num; i++) { + hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN, + GFP_KERNEL); + if (!hr_dev->irq_names[i]) { + ret = -ENOMEM; + goto err_failed_kzalloc; + } + } + + /* create eq */ + for (j = 0; j < eq_num; j++) { + eq = &eq_table->eq[j]; + eq->hr_dev = hr_dev; + eq->eqn = j; + if (j < comp_num) { + /* CEQ */ + eq_cmd = HNS_ROCE_CMD_CREATE_CEQC; + eq->type_flag = HNS_ROCE_CEQ; + eq->entries = hr_dev->caps.ceqe_depth; + eq->eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE; + eq->irq = hr_dev->irq[j + other_num + aeq_num]; + eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM; + eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL; + } else { + /* AEQ */ + eq_cmd = HNS_ROCE_CMD_CREATE_AEQC; + eq->type_flag = HNS_ROCE_AEQ; + eq->entries = hr_dev->caps.aeqe_depth; + eq->eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE; + eq->irq = hr_dev->irq[j - comp_num + other_num]; + eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM; + eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL; + } + + ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd); + if (ret) { + dev_err(dev, "eq create failed.\n"); + goto err_create_eq_fail; + } + } + + /* enable irq */ + hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE); + + /* irq contains: abnormal + AEQ + CEQ*/ + for (k = 0; k < irq_num; k++) + if (k < other_num) + snprintf((char *)hr_dev->irq_names[k], + HNS_ROCE_INT_NAME_LEN, "hns-abn-%d", k); + else if (k < (other_num + aeq_num)) + snprintf((char *)hr_dev->irq_names[k], + HNS_ROCE_INT_NAME_LEN, "hns-aeq-%d", + k - other_num); + else + snprintf((char *)hr_dev->irq_names[k], + HNS_ROCE_INT_NAME_LEN, "hns-ceq-%d", + k - other_num - aeq_num); + + for (k = 0; k < irq_num; k++) { + if (k < other_num) + ret = request_irq(hr_dev->irq[k], + hns_roce_v2_msix_interrupt_abn, + 0, hr_dev->irq_names[k], hr_dev); + + else if (k < (other_num + comp_num)) + ret = request_irq(eq_table->eq[k - other_num].irq, + hns_roce_v2_msix_interrupt_eq, + 0, hr_dev->irq_names[k + aeq_num], + &eq_table->eq[k - other_num]); + else + ret = request_irq(eq_table->eq[k - other_num].irq, + hns_roce_v2_msix_interrupt_eq, + 0, hr_dev->irq_names[k - comp_num], + &eq_table->eq[k - other_num]); + if (ret) { + dev_err(dev, "Request irq error!\n"); + goto err_request_irq_fail; + } + } + + return 0; + +err_request_irq_fail: + for (k -= 1; k >= 0; k--) + if (k < other_num) + free_irq(hr_dev->irq[k], hr_dev); + else + free_irq(eq_table->eq[k - other_num].irq, + &eq_table->eq[k - other_num]); + +err_create_eq_fail: + for (j -= 1; j >= 0; j--) + hns_roce_v2_free_eq(hr_dev, &eq_table->eq[j]); + +err_failed_kzalloc: + for (i -= 1; i >= 0; i--) + kfree(hr_dev->irq_names[i]); + kfree(eq_table->eq); + + return ret; +} + +static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; + int irq_num; + int eq_num; + int i; + + eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; + irq_num = eq_num + hr_dev->caps.num_other_vectors; + + /* Disable irq */ + hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE); + + for (i = 0; i < hr_dev->caps.num_other_vectors; i++) + free_irq(hr_dev->irq[i], hr_dev); + + for (i = 0; i < eq_num; i++) { + hns_roce_v2_destroy_eqc(hr_dev, i); + + free_irq(eq_table->eq[i].irq, &eq_table->eq[i]); + + hns_roce_v2_free_eq(hr_dev, &eq_table->eq[i]); + } + + for (i = 0; i < irq_num; i++) + kfree(hr_dev->irq_names[i]); + + kfree(eq_table->eq); +} + static const struct hns_roce_hw hns_roce_hw_v2 = { .cmq_init = hns_roce_v2_cmq_init, .cmq_exit = hns_roce_v2_cmq_exit, @@ -3183,6 +4345,8 @@ static const struct hns_roce_hw hns_roce_hw_v2 = { .post_recv = hns_roce_v2_post_recv, .req_notify_cq = hns_roce_v2_req_notify_cq, .poll_cq = hns_roce_v2_poll_cq, + .init_eq = hns_roce_v2_init_eq_table, + .cleanup_eq = hns_roce_v2_cleanup_eq_table, }; static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = { @@ -3197,6 +4361,7 @@ static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev, struct hnae3_handle *handle) { const struct pci_device_id *id; + int i; id = pci_match_id(hns_roce_hw_v2_pci_tbl, hr_dev->pci_dev); if (!id) { @@ -3214,8 +4379,12 @@ static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev, hr_dev->iboe.netdevs[0] = handle->rinfo.netdev; hr_dev->iboe.phy_port[0] = 0; + for (i = 0; i < HNS_ROCE_V2_MAX_IRQ_NUM; i++) + hr_dev->irq[i] = pci_irq_vector(handle->pdev, + i + handle->rinfo.base_vector); + /* cmd issue mode: 0 is poll, 1 is event */ - hr_dev->cmd_mod = 0; + hr_dev->cmd_mod = 1; hr_dev->loop_idc = 0; return 0; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index 04b7a51b8efb..463edab9b719 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -53,6 +53,10 @@ #define HNS_ROCE_V2_MAX_SQ_INLINE 0x20 #define HNS_ROCE_V2_UAR_NUM 256 #define HNS_ROCE_V2_PHY_UAR_NUM 1 +#define HNS_ROCE_V2_MAX_IRQ_NUM 65 +#define HNS_ROCE_V2_COMP_VEC_NUM 63 +#define HNS_ROCE_V2_AEQE_VEC_NUM 1 +#define HNS_ROCE_V2_ABNORMAL_VEC_NUM 1 #define HNS_ROCE_V2_MAX_MTPT_NUM 0x8000 #define HNS_ROCE_V2_MAX_MTT_SEGS 0x1000000 #define HNS_ROCE_V2_MAX_CQE_SEGS 0x1000000 @@ -78,6 +82,8 @@ #define HNS_ROCE_MTT_HOP_NUM 1 #define HNS_ROCE_CQE_HOP_NUM 1 #define HNS_ROCE_PBL_HOP_NUM 2 +#define HNS_ROCE_EQE_HOP_NUM 2 + #define HNS_ROCE_V2_GID_INDEX_NUM 256 #define HNS_ROCE_V2_TABLE_CHUNK_SIZE (1 << 18) @@ -105,6 +111,12 @@ (step_idx == 1 && hop_num == 1) || \ (step_idx == 2 && hop_num == 2)) +enum { + NO_ARMED = 0x0, + REG_NXT_CEQE = 0x2, + REG_NXT_SE_CEQE = 0x3 +}; + #define V2_CQ_DB_REQ_NOT_SOL 0 #define V2_CQ_DB_REQ_NOT 1 @@ -229,6 +241,9 @@ struct hns_roce_v2_cq_context { u32 cqe_report_timer; u32 byte_64_se_cqe_idx; }; +#define HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM 0x0 +#define HNS_ROCE_V2_CQ_DEFAULT_INTERVAL 0x0 + #define V2_CQC_BYTE_4_CQ_ST_S 0 #define V2_CQC_BYTE_4_CQ_ST_M GENMASK(1, 0) @@ -1129,9 +1144,6 @@ struct hns_roce_cmq_desc { u32 data[6]; }; -#define ROCEE_VF_MB_CFG0_REG 0x40 -#define ROCEE_VF_MB_STATUS_REG 0x58 - #define HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS 10000 #define HNS_ROCE_HW_RUN_BIT_SHIFT 31 @@ -1174,4 +1186,178 @@ struct hns_roce_v2_priv { struct hns_roce_v2_cmq cmq; }; +struct hns_roce_eq_context { + u32 byte_4; + u32 byte_8; + u32 byte_12; + u32 eqe_report_timer; + u32 eqe_ba0; + u32 eqe_ba1; + u32 byte_28; + u32 byte_32; + u32 byte_36; + u32 nxt_eqe_ba0; + u32 nxt_eqe_ba1; + u32 rsv[5]; +}; + +#define HNS_ROCE_AEQ_DEFAULT_BURST_NUM 0x0 +#define HNS_ROCE_AEQ_DEFAULT_INTERVAL 0x0 +#define HNS_ROCE_CEQ_DEFAULT_BURST_NUM 0x0 +#define HNS_ROCE_CEQ_DEFAULT_INTERVAL 0x0 + +#define HNS_ROCE_V2_EQ_STATE_INVALID 0 +#define HNS_ROCE_V2_EQ_STATE_VALID 1 +#define HNS_ROCE_V2_EQ_STATE_OVERFLOW 2 +#define HNS_ROCE_V2_EQ_STATE_FAILURE 3 + +#define HNS_ROCE_V2_EQ_OVER_IGNORE_0 0 +#define HNS_ROCE_V2_EQ_OVER_IGNORE_1 1 + +#define HNS_ROCE_V2_EQ_COALESCE_0 0 +#define HNS_ROCE_V2_EQ_COALESCE_1 1 + +#define HNS_ROCE_V2_EQ_FIRED 0 +#define HNS_ROCE_V2_EQ_ARMED 1 +#define HNS_ROCE_V2_EQ_ALWAYS_ARMED 3 + +#define HNS_ROCE_EQ_INIT_EQE_CNT 0 +#define HNS_ROCE_EQ_INIT_PROD_IDX 0 +#define HNS_ROCE_EQ_INIT_REPORT_TIMER 0 +#define HNS_ROCE_EQ_INIT_MSI_IDX 0 +#define HNS_ROCE_EQ_INIT_CONS_IDX 0 +#define HNS_ROCE_EQ_INIT_NXT_EQE_BA 0 + +#define HNS_ROCE_V2_CEQ_CEQE_OWNER_S 31 +#define HNS_ROCE_V2_AEQ_AEQE_OWNER_S 31 + +#define HNS_ROCE_V2_COMP_EQE_NUM 0x1000 +#define HNS_ROCE_V2_ASYNC_EQE_NUM 0x1000 + +#define HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S 0 +#define HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S 1 +#define HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S 2 + +#define HNS_ROCE_EQ_DB_CMD_AEQ 0x0 +#define HNS_ROCE_EQ_DB_CMD_AEQ_ARMED 0x1 +#define HNS_ROCE_EQ_DB_CMD_CEQ 0x2 +#define HNS_ROCE_EQ_DB_CMD_CEQ_ARMED 0x3 + +#define EQ_ENABLE 1 +#define EQ_DISABLE 0 + +#define EQ_REG_OFFSET 0x4 + +#define HNS_ROCE_INT_NAME_LEN 32 +#define HNS_ROCE_V2_EQN_M GENMASK(23, 0) + +#define HNS_ROCE_V2_CONS_IDX_M GENMASK(23, 0) + +#define HNS_ROCE_V2_VF_ABN_INT_EN_S 0 +#define HNS_ROCE_V2_VF_ABN_INT_EN_M GENMASK(0, 0) +#define HNS_ROCE_V2_VF_ABN_INT_ST_M GENMASK(2, 0) +#define HNS_ROCE_V2_VF_ABN_INT_CFG_M GENMASK(2, 0) +#define HNS_ROCE_V2_VF_EVENT_INT_EN_M GENMASK(0, 0) + +/* WORD0 */ +#define HNS_ROCE_EQC_EQ_ST_S 0 +#define HNS_ROCE_EQC_EQ_ST_M GENMASK(1, 0) + +#define HNS_ROCE_EQC_HOP_NUM_S 2 +#define HNS_ROCE_EQC_HOP_NUM_M GENMASK(3, 2) + +#define HNS_ROCE_EQC_OVER_IGNORE_S 4 +#define HNS_ROCE_EQC_OVER_IGNORE_M GENMASK(4, 4) + +#define HNS_ROCE_EQC_COALESCE_S 5 +#define HNS_ROCE_EQC_COALESCE_M GENMASK(5, 5) + +#define HNS_ROCE_EQC_ARM_ST_S 6 +#define HNS_ROCE_EQC_ARM_ST_M GENMASK(7, 6) + +#define HNS_ROCE_EQC_EQN_S 8 +#define HNS_ROCE_EQC_EQN_M GENMASK(15, 8) + +#define HNS_ROCE_EQC_EQE_CNT_S 16 +#define HNS_ROCE_EQC_EQE_CNT_M GENMASK(31, 16) + +/* WORD1 */ +#define HNS_ROCE_EQC_BA_PG_SZ_S 0 +#define HNS_ROCE_EQC_BA_PG_SZ_M GENMASK(3, 0) + +#define HNS_ROCE_EQC_BUF_PG_SZ_S 4 +#define HNS_ROCE_EQC_BUF_PG_SZ_M GENMASK(7, 4) + +#define HNS_ROCE_EQC_PROD_INDX_S 8 +#define HNS_ROCE_EQC_PROD_INDX_M GENMASK(31, 8) + +/* WORD2 */ +#define HNS_ROCE_EQC_MAX_CNT_S 0 +#define HNS_ROCE_EQC_MAX_CNT_M GENMASK(15, 0) + +#define HNS_ROCE_EQC_PERIOD_S 16 +#define HNS_ROCE_EQC_PERIOD_M GENMASK(31, 16) + +/* WORD3 */ +#define HNS_ROCE_EQC_REPORT_TIMER_S 0 +#define HNS_ROCE_EQC_REPORT_TIMER_M GENMASK(31, 0) + +/* WORD4 */ +#define HNS_ROCE_EQC_EQE_BA_L_S 0 +#define HNS_ROCE_EQC_EQE_BA_L_M GENMASK(31, 0) + +/* WORD5 */ +#define HNS_ROCE_EQC_EQE_BA_H_S 0 +#define HNS_ROCE_EQC_EQE_BA_H_M GENMASK(28, 0) + +/* WORD6 */ +#define HNS_ROCE_EQC_SHIFT_S 0 +#define HNS_ROCE_EQC_SHIFT_M GENMASK(7, 0) + +#define HNS_ROCE_EQC_MSI_INDX_S 8 +#define HNS_ROCE_EQC_MSI_INDX_M GENMASK(15, 8) + +#define HNS_ROCE_EQC_CUR_EQE_BA_L_S 16 +#define HNS_ROCE_EQC_CUR_EQE_BA_L_M GENMASK(31, 16) + +/* WORD7 */ +#define HNS_ROCE_EQC_CUR_EQE_BA_M_S 0 +#define HNS_ROCE_EQC_CUR_EQE_BA_M_M GENMASK(31, 0) + +/* WORD8 */ +#define HNS_ROCE_EQC_CUR_EQE_BA_H_S 0 +#define HNS_ROCE_EQC_CUR_EQE_BA_H_M GENMASK(3, 0) + +#define HNS_ROCE_EQC_CONS_INDX_S 8 +#define HNS_ROCE_EQC_CONS_INDX_M GENMASK(31, 8) + +/* WORD9 */ +#define HNS_ROCE_EQC_NXT_EQE_BA_L_S 0 +#define HNS_ROCE_EQC_NXT_EQE_BA_L_M GENMASK(31, 0) + +/* WORD10 */ +#define HNS_ROCE_EQC_NXT_EQE_BA_H_S 0 +#define HNS_ROCE_EQC_NXT_EQE_BA_H_M GENMASK(19, 0) + +#define HNS_ROCE_V2_CEQE_COMP_CQN_S 0 +#define HNS_ROCE_V2_CEQE_COMP_CQN_M GENMASK(23, 0) + +#define HNS_ROCE_V2_AEQE_EVENT_TYPE_S 0 +#define HNS_ROCE_V2_AEQE_EVENT_TYPE_M GENMASK(7, 0) + +#define HNS_ROCE_V2_AEQE_SUB_TYPE_S 8 +#define HNS_ROCE_V2_AEQE_SUB_TYPE_M GENMASK(15, 8) + +#define HNS_ROCE_V2_EQ_DB_CMD_S 16 +#define HNS_ROCE_V2_EQ_DB_CMD_M GENMASK(17, 16) + +#define HNS_ROCE_V2_EQ_DB_TAG_S 0 +#define HNS_ROCE_V2_EQ_DB_TAG_M GENMASK(7, 0) + +#define HNS_ROCE_V2_EQ_DB_PARA_S 0 +#define HNS_ROCE_V2_EQ_DB_PARA_M GENMASK(23, 0) + +#define HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S 0 +#define HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M GENMASK(23, 0) + #endif diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index cf02ac2d3596..aa0c242ddc50 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -748,12 +748,10 @@ int hns_roce_init(struct hns_roce_dev *hr_dev) goto error_failed_cmd_init; } - if (hr_dev->cmd_mod) { - ret = hns_roce_init_eq_table(hr_dev); - if (ret) { - dev_err(dev, "eq init failed!\n"); - goto error_failed_eq_table; - } + ret = hr_dev->hw->init_eq(hr_dev); + if (ret) { + dev_err(dev, "eq init failed!\n"); + goto error_failed_eq_table; } if (hr_dev->cmd_mod) { @@ -805,8 +803,7 @@ error_failed_init_hem: hns_roce_cmd_use_polling(hr_dev); error_failed_use_event: - if (hr_dev->cmd_mod) - hns_roce_cleanup_eq_table(hr_dev); + hr_dev->hw->cleanup_eq(hr_dev); error_failed_eq_table: hns_roce_cmd_cleanup(hr_dev); @@ -837,8 +834,7 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev) if (hr_dev->cmd_mod) hns_roce_cmd_use_polling(hr_dev); - if (hr_dev->cmd_mod) - hns_roce_cleanup_eq_table(hr_dev); + hr_dev->hw->cleanup_eq(hr_dev); hns_roce_cmd_cleanup(hr_dev); if (hr_dev->hw->cmq_exit) hr_dev->hw->cmq_exit(hr_dev); diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 49586ec8126a..69e25847ff1b 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -65,6 +65,7 @@ void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) if (atomic_dec_and_test(&qp->refcount)) complete(&qp->free); } +EXPORT_SYMBOL_GPL(hns_roce_qp_event); static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp, enum hns_roce_event type) diff --git a/drivers/infiniband/hw/i40iw/Kconfig b/drivers/infiniband/hw/i40iw/Kconfig index f6d20ba88c03..2962979c06e9 100644 --- a/drivers/infiniband/hw/i40iw/Kconfig +++ b/drivers/infiniband/hw/i40iw/Kconfig @@ -5,4 +5,3 @@ config INFINIBAND_I40IW select GENERIC_ALLOCATOR ---help--- Intel(R) Ethernet X722 iWARP Driver - INET && I40IW && INFINIBAND && I40E diff --git a/drivers/infiniband/hw/i40iw/i40iw.h b/drivers/infiniband/hw/i40iw/i40iw.h index 4ae9131b6350..bcddd7061fc0 100644 --- a/drivers/infiniband/hw/i40iw/i40iw.h +++ b/drivers/infiniband/hw/i40iw/i40iw.h @@ -587,5 +587,8 @@ int i40iw_inet6addr_event(struct notifier_block *notifier, int i40iw_net_event(struct notifier_block *notifier, unsigned long event, void *ptr); +int i40iw_netdevice_event(struct notifier_block *notifier, + unsigned long event, + void *ptr); #endif diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.c b/drivers/infiniband/hw/i40iw/i40iw_cm.c index 0d4c2f430288..abf4cd897849 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_cm.c +++ b/drivers/infiniband/hw/i40iw/i40iw_cm.c @@ -92,14 +92,9 @@ void i40iw_free_sqbuf(struct i40iw_sc_vsi *vsi, void *bufp) static u8 i40iw_derive_hw_ird_setting(u16 cm_ird) { u8 encoded_ird_size; - u8 pof2_cm_ird = 1; - - /* round-off to next powerof2 */ - while (pof2_cm_ird < cm_ird) - pof2_cm_ird *= 2; /* ird_size field is encoded in qp_ctx */ - switch (pof2_cm_ird) { + switch (cm_ird ? roundup_pow_of_two(cm_ird) : 0) { case I40IW_HW_IRD_SETTING_64: encoded_ird_size = 3; break; @@ -125,13 +120,16 @@ static u8 i40iw_derive_hw_ird_setting(u16 cm_ird) * @conn_ird: connection IRD * @conn_ord: connection ORD */ -static void i40iw_record_ird_ord(struct i40iw_cm_node *cm_node, u16 conn_ird, u16 conn_ord) +static void i40iw_record_ird_ord(struct i40iw_cm_node *cm_node, u32 conn_ird, + u32 conn_ord) { if (conn_ird > I40IW_MAX_IRD_SIZE) conn_ird = I40IW_MAX_IRD_SIZE; if (conn_ord > I40IW_MAX_ORD_SIZE) conn_ord = I40IW_MAX_ORD_SIZE; + else if (!conn_ord && cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO) + conn_ord = 1; cm_node->ird_size = conn_ird; cm_node->ord_size = conn_ord; @@ -2878,15 +2876,13 @@ static struct i40iw_cm_listener *i40iw_make_listen_node( * i40iw_create_cm_node - make a connection node with params * @cm_core: cm's core * @iwdev: iwarp device structure - * @private_data_len: len to provate data for mpa request - * @private_data: pointer to private data for connection + * @conn_param: upper layer connection parameters * @cm_info: quad info for connection */ static struct i40iw_cm_node *i40iw_create_cm_node( struct i40iw_cm_core *cm_core, struct i40iw_device *iwdev, - u16 private_data_len, - void *private_data, + struct iw_cm_conn_param *conn_param, struct i40iw_cm_info *cm_info) { struct i40iw_cm_node *cm_node; @@ -2894,6 +2890,9 @@ static struct i40iw_cm_node *i40iw_create_cm_node( struct i40iw_cm_node *loopback_remotenode; struct i40iw_cm_info loopback_cm_info; + u16 private_data_len = conn_param->private_data_len; + const void *private_data = conn_param->private_data; + /* create a CM connection node */ cm_node = i40iw_make_cm_node(cm_core, iwdev, cm_info, NULL); if (!cm_node) @@ -2902,6 +2901,8 @@ static struct i40iw_cm_node *i40iw_create_cm_node( cm_node->tcp_cntxt.client = 1; cm_node->tcp_cntxt.rcv_wscale = I40IW_CM_DEFAULT_RCV_WND_SCALE; + i40iw_record_ird_ord(cm_node, conn_param->ird, conn_param->ord); + if (!memcmp(cm_info->loc_addr, cm_info->rem_addr, sizeof(cm_info->loc_addr))) { loopback_remotelistener = i40iw_find_listener( cm_core, @@ -2935,6 +2936,10 @@ static struct i40iw_cm_node *i40iw_create_cm_node( private_data_len); loopback_remotenode->pdata.size = private_data_len; + if (loopback_remotenode->ord_size > cm_node->ird_size) + loopback_remotenode->ord_size = + cm_node->ird_size; + cm_node->state = I40IW_CM_STATE_OFFLOADED; cm_node->tcp_cntxt.rcv_nxt = loopback_remotenode->tcp_cntxt.loc_seq_num; @@ -3815,9 +3820,7 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) __func__, cm_id->tos, cm_info.user_pri); cm_id->add_ref(cm_id); cm_node = i40iw_create_cm_node(&iwdev->cm_core, iwdev, - conn_param->private_data_len, - (void *)conn_param->private_data, - &cm_info); + conn_param, &cm_info); if (IS_ERR(cm_node)) { ret = PTR_ERR(cm_node); @@ -3849,11 +3852,6 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) } cm_node->apbvt_set = true; - i40iw_record_ird_ord(cm_node, (u16)conn_param->ird, (u16)conn_param->ord); - if (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO && - !cm_node->ord_size) - cm_node->ord_size = 1; - iwqp->cm_node = cm_node; cm_node->iwqp = iwqp; iwqp->cm_id = cm_id; @@ -4242,10 +4240,16 @@ set_qhash: } /** - * i40iw_cm_disconnect_all - disconnect all connected qp's + * i40iw_cm_teardown_connections - teardown QPs * @iwdev: device pointer + * @ipaddr: Pointer to IPv4 or IPv6 address + * @ipv4: flag indicating IPv4 when true + * @disconnect_all: flag indicating disconnect all QPs + * teardown QPs where source or destination addr matches ip addr */ -void i40iw_cm_disconnect_all(struct i40iw_device *iwdev) +void i40iw_cm_teardown_connections(struct i40iw_device *iwdev, u32 *ipaddr, + struct i40iw_cm_info *nfo, + bool disconnect_all) { struct i40iw_cm_core *cm_core = &iwdev->cm_core; struct list_head *list_core_temp; @@ -4259,8 +4263,13 @@ void i40iw_cm_disconnect_all(struct i40iw_device *iwdev) spin_lock_irqsave(&cm_core->ht_lock, flags); list_for_each_safe(list_node, list_core_temp, &cm_core->connected_nodes) { cm_node = container_of(list_node, struct i40iw_cm_node, list); - atomic_inc(&cm_node->ref_count); - list_add(&cm_node->connected_entry, &connected_list); + if (disconnect_all || + (nfo->vlan_id == cm_node->vlan_id && + (!memcmp(cm_node->loc_addr, ipaddr, nfo->ipv4 ? 4 : 16) || + !memcmp(cm_node->rem_addr, ipaddr, nfo->ipv4 ? 4 : 16)))) { + atomic_inc(&cm_node->ref_count); + list_add(&cm_node->connected_entry, &connected_list); + } } spin_unlock_irqrestore(&cm_core->ht_lock, flags); @@ -4294,6 +4303,9 @@ void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev, enum i40iw_quad_hash_manage_type op = ifup ? I40IW_QHASH_MANAGE_TYPE_ADD : I40IW_QHASH_MANAGE_TYPE_DELETE; + nfo.vlan_id = vlan_id; + nfo.ipv4 = ipv4; + /* Disable or enable qhash for listeners */ spin_lock_irqsave(&cm_core->listen_list_lock, flags); list_for_each_entry(listen_node, &cm_core->listen_nodes, list) { @@ -4303,8 +4315,6 @@ void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev, memcpy(nfo.loc_addr, listen_node->loc_addr, sizeof(nfo.loc_addr)); nfo.loc_port = listen_node->loc_port; - nfo.ipv4 = listen_node->ipv4; - nfo.vlan_id = listen_node->vlan_id; nfo.user_pri = listen_node->user_pri; if (!list_empty(&listen_node->child_listen_list)) { i40iw_qhash_ctrl(iwdev, @@ -4326,7 +4336,7 @@ void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev, } spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - /* disconnect any connected qp's on ifdown */ + /* teardown connected qp's on ifdown */ if (!ifup) - i40iw_cm_disconnect_all(iwdev); + i40iw_cm_teardown_connections(iwdev, ipaddr, &nfo, false); } diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.h b/drivers/infiniband/hw/i40iw/i40iw_cm.h index e09b1f5ecf19..cf60c451e071 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_cm.h +++ b/drivers/infiniband/hw/i40iw/i40iw_cm.h @@ -276,8 +276,6 @@ struct i40iw_cm_tcp_context { u32 mss; u8 snd_wscale; u8 rcv_wscale; - - struct timeval sent_ts; }; enum i40iw_cm_listener_state { @@ -455,5 +453,7 @@ int i40iw_arp_table(struct i40iw_device *iwdev, void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev, u32 *ipaddr, bool ipv4, bool ifup); -void i40iw_cm_disconnect_all(struct i40iw_device *iwdev); +void i40iw_cm_teardown_connections(struct i40iw_device *iwdev, u32 *ipaddr, + struct i40iw_cm_info *nfo, + bool disconnect_all); #endif /* I40IW_CM_H */ diff --git a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c index da9821a10e0d..caa958177631 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c +++ b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c @@ -3872,7 +3872,6 @@ enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_ struct i40iw_virt_mem virt_mem; u32 i, mem_size; u32 qpwantedoriginal, qpwanted, mrwanted, pblewanted; - u32 powerof2; u64 sd_needed; u32 loop_count = 0; @@ -3928,8 +3927,10 @@ enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_ hmc_info->hmc_obj[I40IW_HMC_IW_APBVT_ENTRY].cnt = 1; hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt = mrwanted; - hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt = I40IW_MAX_WQ_ENTRIES * qpwanted; - hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt = 4 * I40IW_MAX_IRD_SIZE * qpwanted; + hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt = + roundup_pow_of_two(I40IW_MAX_WQ_ENTRIES * qpwanted); + hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt = + roundup_pow_of_two(2 * I40IW_MAX_IRD_SIZE * qpwanted); hmc_info->hmc_obj[I40IW_HMC_IW_XFFL].cnt = hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size; hmc_info->hmc_obj[I40IW_HMC_IW_Q1FL].cnt = @@ -3945,16 +3946,10 @@ enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_ if ((loop_count > 1000) || ((!(loop_count % 10)) && (qpwanted > qpwantedoriginal * 2 / 3))) { - if (qpwanted > FPM_MULTIPLIER) { - qpwanted -= FPM_MULTIPLIER; - powerof2 = 1; - while (powerof2 < qpwanted) - powerof2 *= 2; - powerof2 /= 2; - qpwanted = powerof2; - } else { - qpwanted /= 2; - } + if (qpwanted > FPM_MULTIPLIER) + qpwanted = roundup_pow_of_two(qpwanted - + FPM_MULTIPLIER); + qpwanted >>= 1; } if (mrwanted > FPM_MULTIPLIER * 10) mrwanted -= FPM_MULTIPLIER * 10; diff --git a/drivers/infiniband/hw/i40iw/i40iw_d.h b/drivers/infiniband/hw/i40iw/i40iw_d.h index 029083cb81d5..4b65e4140bd7 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_d.h +++ b/drivers/infiniband/hw/i40iw/i40iw_d.h @@ -97,6 +97,7 @@ #define RDMA_OPCODE_MASK 0x0f #define RDMA_READ_REQ_OPCODE 1 #define Q2_BAD_FRAME_OFFSET 72 +#define Q2_FPSN_OFFSET 64 #define CQE_MAJOR_DRV 0x8000 #define I40IW_TERM_SENT 0x01 diff --git a/drivers/infiniband/hw/i40iw/i40iw_hw.c b/drivers/infiniband/hw/i40iw/i40iw_hw.c index e96bdafbcbb3..61540e14e4b9 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_hw.c +++ b/drivers/infiniband/hw/i40iw/i40iw_hw.c @@ -385,6 +385,8 @@ void i40iw_process_aeq(struct i40iw_device *iwdev) iwcq->ibcq.event_handler(&ibevent, iwcq->ibcq.cq_context); } break; + case I40IW_AE_LLP_DOUBT_REACHABILITY: + break; case I40IW_AE_PRIV_OPERATION_DENIED: case I40IW_AE_STAG_ZERO_INVALID: case I40IW_AE_IB_RREQ_AND_Q1_FULL: @@ -403,7 +405,6 @@ void i40iw_process_aeq(struct i40iw_device *iwdev) case I40IW_AE_LLP_SEGMENT_TOO_SMALL: case I40IW_AE_LLP_SYN_RECEIVED: case I40IW_AE_LLP_TOO_MANY_RETRIES: - case I40IW_AE_LLP_DOUBT_REACHABILITY: case I40IW_AE_LCE_QP_CATASTROPHIC: case I40IW_AE_LCE_FUNCTION_CATASTROPHIC: case I40IW_AE_LCE_CQ_CATASTROPHIC: diff --git a/drivers/infiniband/hw/i40iw/i40iw_main.c b/drivers/infiniband/hw/i40iw/i40iw_main.c index e824296713e2..8a9815e43282 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_main.c +++ b/drivers/infiniband/hw/i40iw/i40iw_main.c @@ -99,6 +99,10 @@ static struct notifier_block i40iw_net_notifier = { .notifier_call = i40iw_net_event }; +static struct notifier_block i40iw_netdevice_notifier = { + .notifier_call = i40iw_netdevice_event +}; + /** * i40iw_find_i40e_handler - find a handler given a client info * @ldev: pointer to a client info @@ -483,6 +487,7 @@ static enum i40iw_status_code i40iw_create_hmc_objs(struct i40iw_device *iwdev, for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++) { info.rsrc_type = iw_hmc_obj_types[i]; info.count = dev->hmc_info->hmc_obj[info.rsrc_type].cnt; + info.add_sd_cnt = 0; status = i40iw_create_hmc_obj_type(dev, &info); if (status) { i40iw_pr_err("create obj type %d status = %d\n", @@ -607,7 +612,7 @@ static enum i40iw_status_code i40iw_create_cqp(struct i40iw_device *iwdev) INIT_LIST_HEAD(&cqp->cqp_avail_reqs); INIT_LIST_HEAD(&cqp->cqp_pending_reqs); /* init the waitq of the cqp_requests and add them to the list */ - for (i = 0; i < I40IW_CQP_SW_SQSIZE_2048; i++) { + for (i = 0; i < sqsize; i++) { init_waitqueue_head(&cqp->cqp_requests[i].waitq); list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs); } @@ -1393,6 +1398,7 @@ static void i40iw_register_notifiers(void) register_inetaddr_notifier(&i40iw_inetaddr_notifier); register_inet6addr_notifier(&i40iw_inetaddr6_notifier); register_netevent_notifier(&i40iw_net_notifier); + register_netdevice_notifier(&i40iw_netdevice_notifier); } /** @@ -1404,6 +1410,7 @@ static void i40iw_unregister_notifiers(void) unregister_netevent_notifier(&i40iw_net_notifier); unregister_inetaddr_notifier(&i40iw_inetaddr_notifier); unregister_inet6addr_notifier(&i40iw_inetaddr6_notifier); + unregister_netdevice_notifier(&i40iw_netdevice_notifier); } /** @@ -1793,7 +1800,7 @@ static void i40iw_close(struct i40e_info *ldev, struct i40e_client *client, bool if (reset) iwdev->reset = true; - i40iw_cm_disconnect_all(iwdev); + i40iw_cm_teardown_connections(iwdev, NULL, NULL, true); destroy_workqueue(iwdev->virtchnl_wq); i40iw_deinit_device(iwdev); } diff --git a/drivers/infiniband/hw/i40iw/i40iw_puda.c b/drivers/infiniband/hw/i40iw/i40iw_puda.c index 796a815b53fd..f64b6700f43f 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_puda.c +++ b/drivers/infiniband/hw/i40iw/i40iw_puda.c @@ -1378,7 +1378,7 @@ static void i40iw_ieq_handle_exception(struct i40iw_puda_rsrc *ieq, u32 *hw_host_ctx = (u32 *)qp->hw_host_ctx; u32 rcv_wnd = hw_host_ctx[23]; /* first partial seq # in q2 */ - u32 fps = qp->q2_buf[16]; + u32 fps = *(u32 *)(qp->q2_buf + Q2_FPSN_OFFSET); struct list_head *rxlist = &pfpdu->rxlist; struct list_head *plist; diff --git a/drivers/infiniband/hw/i40iw/i40iw_uk.c b/drivers/infiniband/hw/i40iw/i40iw_uk.c index 3ec5389a81a1..8afa5a67a86b 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_uk.c +++ b/drivers/infiniband/hw/i40iw/i40iw_uk.c @@ -894,20 +894,6 @@ exit: } /** - * i40iw_qp_roundup - return round up QP WQ depth - * @wqdepth: WQ depth in quantas to round up - */ -static int i40iw_qp_round_up(u32 wqdepth) -{ - int scount = 1; - - for (wqdepth--; scount <= 16; scount *= 2) - wqdepth |= wqdepth >> scount; - - return ++wqdepth; -} - -/** * i40iw_get_wqe_shift - get shift count for maximum wqe size * @sge: Maximum Scatter Gather Elements wqe * @inline_data: Maximum inline data size @@ -934,7 +920,7 @@ void i40iw_get_wqe_shift(u32 sge, u32 inline_data, u8 *shift) */ enum i40iw_status_code i40iw_get_sqdepth(u32 sq_size, u8 shift, u32 *sqdepth) { - *sqdepth = i40iw_qp_round_up((sq_size << shift) + I40IW_SQ_RSVD); + *sqdepth = roundup_pow_of_two((sq_size << shift) + I40IW_SQ_RSVD); if (*sqdepth < (I40IW_QP_SW_MIN_WQSIZE << shift)) *sqdepth = I40IW_QP_SW_MIN_WQSIZE << shift; @@ -953,7 +939,7 @@ enum i40iw_status_code i40iw_get_sqdepth(u32 sq_size, u8 shift, u32 *sqdepth) */ enum i40iw_status_code i40iw_get_rqdepth(u32 rq_size, u8 shift, u32 *rqdepth) { - *rqdepth = i40iw_qp_round_up((rq_size << shift) + I40IW_RQ_RSVD); + *rqdepth = roundup_pow_of_two((rq_size << shift) + I40IW_RQ_RSVD); if (*rqdepth < (I40IW_QP_SW_MIN_WQSIZE << shift)) *rqdepth = I40IW_QP_SW_MIN_WQSIZE << shift; diff --git a/drivers/infiniband/hw/i40iw/i40iw_user.h b/drivers/infiniband/hw/i40iw/i40iw_user.h index e73efc59a0ab..5467c6fdad03 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_user.h +++ b/drivers/infiniband/hw/i40iw/i40iw_user.h @@ -72,7 +72,7 @@ enum i40iw_device_capabilities_const { I40IW_MAX_SQ_PAYLOAD_SIZE = 2145386496, I40IW_MAX_INLINE_DATA_SIZE = 48, I40IW_MAX_PUSHMODE_INLINE_DATA_SIZE = 48, - I40IW_MAX_IRD_SIZE = 63, + I40IW_MAX_IRD_SIZE = 64, I40IW_MAX_ORD_SIZE = 127, I40IW_MAX_WQ_ENTRIES = 2048, I40IW_Q2_BUFFER_SIZE = (248 + 100), diff --git a/drivers/infiniband/hw/i40iw/i40iw_utils.c b/drivers/infiniband/hw/i40iw/i40iw_utils.c index 8845dba7c438..ddc1056b0b4e 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_utils.c +++ b/drivers/infiniband/hw/i40iw/i40iw_utils.c @@ -137,7 +137,7 @@ inline u32 i40iw_rd32(struct i40iw_hw *hw, u32 reg) } /** - * i40iw_inetaddr_event - system notifier for netdev events + * i40iw_inetaddr_event - system notifier for ipv4 addr events * @notfier: not used * @event: event for notifier * @ptr: if address @@ -200,7 +200,7 @@ int i40iw_inetaddr_event(struct notifier_block *notifier, } /** - * i40iw_inet6addr_event - system notifier for ipv6 netdev events + * i40iw_inet6addr_event - system notifier for ipv6 addr events * @notfier: not used * @event: event for notifier * @ptr: if address @@ -252,7 +252,7 @@ int i40iw_inet6addr_event(struct notifier_block *notifier, } /** - * i40iw_net_event - system notifier for net events + * i40iw_net_event - system notifier for netevents * @notfier: not used * @event: event for notifier * @ptr: neighbor @@ -297,6 +297,50 @@ int i40iw_net_event(struct notifier_block *notifier, unsigned long event, void * } /** + * i40iw_netdevice_event - system notifier for netdev events + * @notfier: not used + * @event: event for notifier + * @ptr: netdev + */ +int i40iw_netdevice_event(struct notifier_block *notifier, + unsigned long event, + void *ptr) +{ + struct net_device *event_netdev; + struct net_device *netdev; + struct i40iw_device *iwdev; + struct i40iw_handler *hdl; + + event_netdev = netdev_notifier_info_to_dev(ptr); + + hdl = i40iw_find_netdev(event_netdev); + if (!hdl) + return NOTIFY_DONE; + + iwdev = &hdl->device; + if (iwdev->init_state < RDMA_DEV_REGISTERED || iwdev->closing) + return NOTIFY_DONE; + + netdev = iwdev->ldev->netdev; + if (netdev != event_netdev) + return NOTIFY_DONE; + + iwdev->iw_status = 1; + + switch (event) { + case NETDEV_DOWN: + iwdev->iw_status = 0; + /* Fall through */ + case NETDEV_UP: + i40iw_port_ibevent(iwdev); + break; + default: + break; + } + return NOTIFY_DONE; +} + +/** * i40iw_get_cqp_request - get cqp struct * @cqp: device cqp ptr * @wait: cqp to be used in wait mode diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index caf490ab24c8..f5f5c5960bba 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -737,7 +737,7 @@ static int set_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_rss *rss_ctx, return 0; } -static int create_qp_rss(struct mlx4_ib_dev *dev, struct ib_pd *ibpd, +static int create_qp_rss(struct mlx4_ib_dev *dev, struct ib_qp_init_attr *init_attr, struct mlx4_ib_create_qp_rss *ucmd, struct mlx4_ib_qp *qp) @@ -860,7 +860,7 @@ static struct ib_qp *_mlx4_ib_create_qp_rss(struct ib_pd *pd, qp->pri.vid = 0xFFFF; qp->alt.vid = 0xFFFF; - err = create_qp_rss(to_mdev(pd->device), pd, init_attr, &ucmd, qp); + err = create_qp_rss(to_mdev(pd->device), init_attr, &ucmd, qp); if (err) { kfree(qp); return ERR_PTR(err); @@ -1836,6 +1836,8 @@ static int _mlx4_set_path(struct mlx4_ib_dev *dev, mlx4_ib_gid_index_to_real_index(dev, port, grh->sgid_index); + if (real_sgid_index < 0) + return real_sgid_index; if (real_sgid_index >= dev->dev->caps.gid_table_len[port]) { pr_err("sgid_index (%u) too large. max is %d\n", real_sgid_index, dev->dev->caps.gid_table_len[port] - 1); diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 8ac50de2b242..46c7367d3255 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -50,16 +50,14 @@ #include <rdma/ib_cache.h> #include <linux/mlx5/port.h> #include <linux/mlx5/vport.h> +#include <linux/mlx5/fs.h> #include <linux/list.h> #include <rdma/ib_smi.h> #include <rdma/ib_umem.h> #include <linux/in.h> #include <linux/etherdevice.h> -#include <linux/mlx5/fs.h> -#include <linux/mlx5/vport.h> #include "mlx5_ib.h" #include "cmd.h" -#include <linux/mlx5/vport.h> #define DRIVER_NAME "mlx5_ib" #define DRIVER_VERSION "5.0-0" diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index d109fe8290a7..556e015678de 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1206,6 +1206,9 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, int err; bool use_umr = true; + if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM)) + return ERR_PTR(-EINVAL); + mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n", start, virt_addr, length, access_flags); diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c index c56ca2a74df5..6cdfbf8c5674 100644 --- a/drivers/infiniband/hw/nes/nes_cm.c +++ b/drivers/infiniband/hw/nes/nes_cm.c @@ -1365,7 +1365,7 @@ static int mini_cm_del_listen(struct nes_cm_core *cm_core, static inline int mini_cm_accelerated(struct nes_cm_core *cm_core, struct nes_cm_node *cm_node) { - cm_node->accelerated = 1; + cm_node->accelerated = true; if (cm_node->accept_pend) { BUG_ON(!cm_node->listener); diff --git a/drivers/infiniband/hw/nes/nes_cm.h b/drivers/infiniband/hw/nes/nes_cm.h index d827d03e3941..b9cc02b4e8d5 100644 --- a/drivers/infiniband/hw/nes/nes_cm.h +++ b/drivers/infiniband/hw/nes/nes_cm.h @@ -279,7 +279,6 @@ struct nes_cm_tcp_context { u8 rcv_wscale; struct nes_cm_tsa_context tsa_cntxt; - struct timeval sent_ts; }; @@ -341,7 +340,7 @@ struct nes_cm_node { u16 mpa_frame_size; struct iw_cm_id *cm_id; struct list_head list; - int accelerated; + bool accelerated; struct nes_cm_listener *listener; enum nes_cm_conn_type conn_type; struct nes_vnic *nesvnic; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_stats.c b/drivers/infiniband/hw/ocrdma/ocrdma_stats.c index e528d7acb7f6..fb78b16ce671 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_stats.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_stats.c @@ -834,7 +834,7 @@ void ocrdma_add_port_stats(struct ocrdma_dev *dev) dev->reset_stats.type = OCRDMA_RESET_STATS; dev->reset_stats.dev = dev; - if (!debugfs_create_file("reset_stats", S_IRUSR, dev->dir, + if (!debugfs_create_file("reset_stats", 0200, dev->dir, &dev->reset_stats, &ocrdma_dbg_ops)) goto err; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index 7866fd8051f6..b5f1dd33b73e 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -461,7 +461,7 @@ retry: static inline int is_ucontext_pd(struct ocrdma_ucontext *uctx, struct ocrdma_pd *pd) { - return (uctx->cntxt_pd == pd ? true : false); + return (uctx->cntxt_pd == pd); } static int _ocrdma_dealloc_pd(struct ocrdma_dev *dev, diff --git a/drivers/infiniband/hw/qib/qib_file_ops.c b/drivers/infiniband/hw/qib/qib_file_ops.c index 2d6a191afec0..a9a48b393323 100644 --- a/drivers/infiniband/hw/qib/qib_file_ops.c +++ b/drivers/infiniband/hw/qib/qib_file_ops.c @@ -568,20 +568,16 @@ done: static int qib_set_part_key(struct qib_ctxtdata *rcd, u16 key) { struct qib_pportdata *ppd = rcd->ppd; - int i, any = 0, pidx = -1; + int i, pidx = -1; + bool any = false; u16 lkey = key & 0x7FFF; - int ret; - if (lkey == (QIB_DEFAULT_P_KEY & 0x7FFF)) { + if (lkey == (QIB_DEFAULT_P_KEY & 0x7FFF)) /* nothing to do; this key always valid */ - ret = 0; - goto bail; - } + return 0; - if (!lkey) { - ret = -EINVAL; - goto bail; - } + if (!lkey) + return -EINVAL; /* * Set the full membership bit, because it has to be @@ -594,18 +590,14 @@ static int qib_set_part_key(struct qib_ctxtdata *rcd, u16 key) for (i = 0; i < ARRAY_SIZE(rcd->pkeys); i++) { if (!rcd->pkeys[i] && pidx == -1) pidx = i; - if (rcd->pkeys[i] == key) { - ret = -EEXIST; - goto bail; - } + if (rcd->pkeys[i] == key) + return -EEXIST; } - if (pidx == -1) { - ret = -EBUSY; - goto bail; - } - for (any = i = 0; i < ARRAY_SIZE(ppd->pkeys); i++) { + if (pidx == -1) + return -EBUSY; + for (i = 0; i < ARRAY_SIZE(ppd->pkeys); i++) { if (!ppd->pkeys[i]) { - any++; + any = true; continue; } if (ppd->pkeys[i] == key) { @@ -613,44 +605,34 @@ static int qib_set_part_key(struct qib_ctxtdata *rcd, u16 key) if (atomic_inc_return(pkrefs) > 1) { rcd->pkeys[pidx] = key; - ret = 0; - goto bail; - } else { - /* - * lost race, decrement count, catch below - */ - atomic_dec(pkrefs); - any++; + return 0; } + /* + * lost race, decrement count, catch below + */ + atomic_dec(pkrefs); + any = true; } - if ((ppd->pkeys[i] & 0x7FFF) == lkey) { + if ((ppd->pkeys[i] & 0x7FFF) == lkey) /* * It makes no sense to have both the limited and * full membership PKEY set at the same time since * the unlimited one will disable the limited one. */ - ret = -EEXIST; - goto bail; - } - } - if (!any) { - ret = -EBUSY; - goto bail; + return -EEXIST; } - for (any = i = 0; i < ARRAY_SIZE(ppd->pkeys); i++) { + if (!any) + return -EBUSY; + for (i = 0; i < ARRAY_SIZE(ppd->pkeys); i++) { if (!ppd->pkeys[i] && atomic_inc_return(&ppd->pkeyrefs[i]) == 1) { rcd->pkeys[pidx] = key; ppd->pkeys[i] = key; (void) ppd->dd->f_set_ib_cfg(ppd, QIB_IB_CFG_PKEYS, 0); - ret = 0; - goto bail; + return 0; } } - ret = -EBUSY; - -bail: - return ret; + return -EBUSY; } /** diff --git a/drivers/infiniband/hw/qib/qib_rc.c b/drivers/infiniband/hw/qib/qib_rc.c index 8f5754fb8579..e4a9ba1dd9ba 100644 --- a/drivers/infiniband/hw/qib/qib_rc.c +++ b/drivers/infiniband/hw/qib/qib_rc.c @@ -434,13 +434,13 @@ no_flow_control: qp->s_state = OP(COMPARE_SWAP); put_ib_ateth_swap(wqe->atomic_wr.swap, &ohdr->u.atomic_eth); - put_ib_ateth_swap(wqe->atomic_wr.compare_add, - &ohdr->u.atomic_eth); + put_ib_ateth_compare(wqe->atomic_wr.compare_add, + &ohdr->u.atomic_eth); } else { qp->s_state = OP(FETCH_ADD); put_ib_ateth_swap(wqe->atomic_wr.compare_add, &ohdr->u.atomic_eth); - put_ib_ateth_swap(0, &ohdr->u.atomic_eth); + put_ib_ateth_compare(0, &ohdr->u.atomic_eth); } put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr, &ohdr->u.atomic_eth); diff --git a/drivers/infiniband/hw/usnic/usnic_ib_sysfs.c b/drivers/infiniband/hw/usnic/usnic_ib_sysfs.c index 685ef2293cb8..4210ca14014d 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_sysfs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_sysfs.c @@ -45,7 +45,6 @@ #include "usnic_ib_verbs.h" #include "usnic_ib_sysfs.h" #include "usnic_log.h" -#include "usnic_ib_sysfs.h" static ssize_t usnic_ib_show_board(struct device *device, struct device_attribute *attr, diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c index aa2456a4f9bd..a688a5669168 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c @@ -47,7 +47,6 @@ #include "usnic_log.h" #include "usnic_uiom.h" #include "usnic_transport.h" -#include "usnic_ib_verbs.h" #define USNIC_DEFAULT_TRANSPORT USNIC_TRANSPORT_ROCE_CUSTOM diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c index 8519f3212e52..fa96fa4fb829 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c @@ -119,10 +119,7 @@ struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, union pvrdma_cmd_resp rsp; struct pvrdma_cmd_create_mr *cmd = &req.create_mr; struct pvrdma_cmd_create_mr_resp *resp = &rsp.create_mr_resp; - int nchunks; int ret; - int entry; - struct scatterlist *sg; if (length == 0 || length > dev->dsr->caps.max_mr_size) { dev_warn(&dev->pdev->dev, "invalid mem region length\n"); @@ -137,13 +134,9 @@ struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, return ERR_CAST(umem); } - nchunks = 0; - for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) - nchunks += sg_dma_len(sg) >> PAGE_SHIFT; - - if (nchunks < 0 || nchunks > PVRDMA_PAGE_DIR_MAX_PAGES) { + if (umem->npages < 0 || umem->npages > PVRDMA_PAGE_DIR_MAX_PAGES) { dev_warn(&dev->pdev->dev, "overflow %d pages in mem region\n", - nchunks); + umem->npages); ret = -EINVAL; goto err_umem; } @@ -158,7 +151,7 @@ struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, mr->mmr.size = length; mr->umem = umem; - ret = pvrdma_page_dir_init(dev, &mr->pdir, nchunks, false); + ret = pvrdma_page_dir_init(dev, &mr->pdir, umem->npages, false); if (ret) { dev_warn(&dev->pdev->dev, "could not allocate page directory\n"); @@ -175,7 +168,7 @@ struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, cmd->length = length; cmd->pd_handle = to_vpd(pd)->pd_handle; cmd->access_flags = access_flags; - cmd->nchunks = nchunks; + cmd->nchunks = umem->npages; cmd->pdir_dma = mr->pdir.dir_dma; ret = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_CREATE_MR_RESP); diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c index fb8c83e055e1..4c3f899241d4 100644 --- a/drivers/infiniband/sw/rxe/rxe_recv.c +++ b/drivers/infiniband/sw/rxe/rxe_recv.c @@ -336,7 +336,6 @@ static int rxe_match_dgid(struct rxe_dev *rxe, struct sk_buff *skb) { union ib_gid dgid; union ib_gid *pdgid; - u16 index; if (skb->protocol == htons(ETH_P_IP)) { ipv6_addr_set_v4mapped(ip_hdr(skb)->daddr, @@ -348,7 +347,7 @@ static int rxe_match_dgid(struct rxe_dev *rxe, struct sk_buff *skb) return ib_find_cached_gid_by_port(&rxe->ib_dev, pdgid, IB_GID_TYPE_ROCE_UDP_ENCAP, - 1, rxe->ndev, &index); + 1, rxe->ndev, NULL); } /* rxe_rcv is called from the interface driver */ diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c index 2c13123bfd69..d45b0f2aa987 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c @@ -876,7 +876,7 @@ int ipoib_cm_dev_open(struct net_device *dev) priv->cm.id = ib_create_cm_id(priv->ca, ipoib_cm_rx_handler, dev); if (IS_ERR(priv->cm.id)) { - printk(KERN_WARNING "%s: failed to create CM ID\n", priv->ca->name); + pr_warn("%s: failed to create CM ID\n", priv->ca->name); ret = PTR_ERR(priv->cm.id); goto err_cm; } @@ -884,8 +884,8 @@ int ipoib_cm_dev_open(struct net_device *dev) ret = ib_cm_listen(priv->cm.id, cpu_to_be64(IPOIB_CM_IETF_ID | priv->qp->qp_num), 0); if (ret) { - printk(KERN_WARNING "%s: failed to listen on ID 0x%llx\n", priv->ca->name, - IPOIB_CM_IETF_ID | priv->qp->qp_num); + pr_warn("%s: failed to listen on ID 0x%llx\n", priv->ca->name, + IPOIB_CM_IETF_ID | priv->qp->qp_num); goto err_listen; } @@ -1563,7 +1563,7 @@ static void ipoib_cm_create_srq(struct net_device *dev, int max_sge) priv->cm.srq = ib_create_srq(priv->pd, &srq_init_attr); if (IS_ERR(priv->cm.srq)) { if (PTR_ERR(priv->cm.srq) != -ENOSYS) - printk(KERN_WARNING "%s: failed to allocate SRQ, error %ld\n", + pr_warn("%s: failed to allocate SRQ, error %ld\n", priv->ca->name, PTR_ERR(priv->cm.srq)); priv->cm.srq = NULL; return; diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c b/drivers/infiniband/ulp/ipoib/ipoib_ib.c index e6151a29c412..a8760d40dfa6 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c @@ -1085,8 +1085,7 @@ static bool ipoib_dev_addr_changed_valid(struct ipoib_dev_priv *priv) netif_addr_unlock_bh(priv->dev); - err = ib_find_gid(priv->ca, &search_gid, IB_GID_TYPE_IB, - priv->dev, &port, &index); + err = ib_find_gid(priv->ca, &search_gid, priv->dev, &port, &index); netif_addr_lock_bh(priv->dev); diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c index 12b7f911f0e5..cb79be1a1474 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -768,13 +768,30 @@ static void path_rec_completion(int status, if (!status) { struct rdma_ah_attr av; - if (!ib_init_ah_from_path(priv->ca, priv->port, pathrec, &av)) + if (!ib_init_ah_attr_from_path(priv->ca, priv->port, + pathrec, &av)) ah = ipoib_create_ah(dev, priv->pd, &av); } spin_lock_irqsave(&priv->lock, flags); if (!IS_ERR_OR_NULL(ah)) { + /* + * pathrec.dgid is used as the database key from the LLADDR, + * it must remain unchanged even if the SA returns a different + * GID to use in the AH. + */ + if (memcmp(pathrec->dgid.raw, path->pathrec.dgid.raw, + sizeof(union ib_gid))) { + ipoib_dbg( + priv, + "%s got PathRec for gid %pI6 while asked for %pI6\n", + dev->name, pathrec->dgid.raw, + path->pathrec.dgid.raw); + memcpy(pathrec->dgid.raw, path->pathrec.dgid.raw, + sizeof(union ib_gid)); + } + path->pathrec = *pathrec; old_ah = path->ah; @@ -840,6 +857,23 @@ static void path_rec_completion(int status, } } +static void init_path_rec(struct ipoib_dev_priv *priv, struct ipoib_path *path, + void *gid) +{ + path->dev = priv->dev; + + if (rdma_cap_opa_ah(priv->ca, priv->port)) + path->pathrec.rec_type = SA_PATH_REC_TYPE_OPA; + else + path->pathrec.rec_type = SA_PATH_REC_TYPE_IB; + + memcpy(path->pathrec.dgid.raw, gid, sizeof(union ib_gid)); + path->pathrec.sgid = priv->local_gid; + path->pathrec.pkey = cpu_to_be16(priv->pkey); + path->pathrec.numb_path = 1; + path->pathrec.traffic_class = priv->broadcast->mcmember.traffic_class; +} + static struct ipoib_path *path_rec_create(struct net_device *dev, void *gid) { struct ipoib_dev_priv *priv = ipoib_priv(dev); @@ -852,21 +886,11 @@ static struct ipoib_path *path_rec_create(struct net_device *dev, void *gid) if (!path) return NULL; - path->dev = dev; - skb_queue_head_init(&path->queue); INIT_LIST_HEAD(&path->neigh_list); - if (rdma_cap_opa_ah(priv->ca, priv->port)) - path->pathrec.rec_type = SA_PATH_REC_TYPE_OPA; - else - path->pathrec.rec_type = SA_PATH_REC_TYPE_IB; - memcpy(path->pathrec.dgid.raw, gid, sizeof (union ib_gid)); - path->pathrec.sgid = priv->local_gid; - path->pathrec.pkey = cpu_to_be16(priv->pkey); - path->pathrec.numb_path = 1; - path->pathrec.traffic_class = priv->broadcast->mcmember.traffic_class; + init_path_rec(priv, path, gid); return path; } @@ -995,6 +1019,10 @@ static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev, spin_lock_irqsave(&priv->lock, flags); + /* no broadcast means that all paths are (going to be) not valid */ + if (!priv->broadcast) + goto drop_and_unlock; + path = __path_find(dev, phdr->hwaddr + 4); if (!path || !path->valid) { int new_path = 0; @@ -1004,6 +1032,10 @@ static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev, new_path = 1; } if (path) { + if (!new_path) + /* make sure there is no changes in the existing path record */ + init_path_rec(priv, path, phdr->hwaddr + 4); + if (skb_queue_len(&path->queue) < IPOIB_MAX_PATH_REC_QUEUE) { push_pseudo_header(skb, phdr->hwaddr); __skb_queue_tail(&path->queue, skb); @@ -1020,8 +1052,7 @@ static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev, } else __path_add(dev, path); } else { - ++dev->stats.tx_dropped; - dev_kfree_skb_any(skb); + goto drop_and_unlock; } spin_unlock_irqrestore(&priv->lock, flags); @@ -1041,11 +1072,16 @@ static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev, push_pseudo_header(skb, phdr->hwaddr); __skb_queue_tail(&path->queue, skb); } else { - ++dev->stats.tx_dropped; - dev_kfree_skb_any(skb); + goto drop_and_unlock; } spin_unlock_irqrestore(&priv->lock, flags); + return; + +drop_and_unlock: + ++dev->stats.tx_dropped; + dev_kfree_skb_any(skb); + spin_unlock_irqrestore(&priv->lock, flags); } static int ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev) @@ -1663,8 +1699,8 @@ static int ipoib_dev_init_default(struct net_device *dev) priv->tx_ring = vzalloc(ipoib_sendq_size * sizeof *priv->tx_ring); if (!priv->tx_ring) { - printk(KERN_WARNING "%s: failed to allocate TX ring (%d entries)\n", - priv->ca->name, ipoib_sendq_size); + pr_warn("%s: failed to allocate TX ring (%d entries)\n", + priv->ca->name, ipoib_sendq_size); goto out_rx_ring_cleanup; } @@ -2196,16 +2232,17 @@ static struct net_device *ipoib_add_port(const char *format, int result = -ENOMEM; priv = ipoib_intf_alloc(hca, port, format); - if (!priv) + if (!priv) { + pr_warn("%s, %d: ipoib_intf_alloc failed\n", hca->name, port); goto alloc_mem_failed; + } SET_NETDEV_DEV(priv->dev, hca->dev.parent); priv->dev->dev_id = port - 1; result = ib_query_port(hca, port, &attr); if (result) { - printk(KERN_WARNING "%s: ib_query_port %d failed\n", - hca->name, port); + pr_warn("%s: ib_query_port %d failed\n", hca->name, port); goto device_init_failed; } @@ -2220,8 +2257,8 @@ static struct net_device *ipoib_add_port(const char *format, result = ib_query_pkey(hca, port, 0, &priv->pkey); if (result) { - printk(KERN_WARNING "%s: ib_query_pkey port %d failed (ret = %d)\n", - hca->name, port, result); + pr_warn("%s: ib_query_pkey port %d failed (ret = %d)\n", + hca->name, port, result); goto device_init_failed; } @@ -2238,8 +2275,8 @@ static struct net_device *ipoib_add_port(const char *format, result = ib_query_gid(hca, port, 0, &priv->local_gid, NULL); if (result) { - printk(KERN_WARNING "%s: ib_query_gid port %d failed (ret = %d)\n", - hca->name, port, result); + pr_warn("%s: ib_query_gid port %d failed (ret = %d)\n", + hca->name, port, result); goto device_init_failed; } @@ -2249,8 +2286,8 @@ static struct net_device *ipoib_add_port(const char *format, result = ipoib_dev_init(priv->dev, hca, port); if (result) { - printk(KERN_WARNING "%s: failed to initialize port %d (ret = %d)\n", - hca->name, port, result); + pr_warn("%s: failed to initialize port %d (ret = %d)\n", + hca->name, port, result); goto device_init_failed; } @@ -2260,8 +2297,8 @@ static struct net_device *ipoib_add_port(const char *format, result = register_netdev(priv->dev); if (result) { - printk(KERN_WARNING "%s: couldn't register ipoib port %d; error %d\n", - hca->name, port, result); + pr_warn("%s: couldn't register ipoib port %d; error %d\n", + hca->name, port, result); goto register_failed; } @@ -2326,8 +2363,7 @@ static void ipoib_add_one(struct ib_device *device) } if (!count) { - pr_err("Failed to init port, removing it\n"); - ipoib_remove_one(device, dev_list); + kfree(dev_list); return; } diff --git a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c index a1ed25422b72..984a88096f39 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c @@ -178,7 +178,7 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca) priv->recv_cq = ib_create_cq(priv->ca, ipoib_ib_rx_completion, NULL, priv, &cq_attr); if (IS_ERR(priv->recv_cq)) { - printk(KERN_WARNING "%s: failed to create receive CQ\n", ca->name); + pr_warn("%s: failed to create receive CQ\n", ca->name); goto out_cm_dev_cleanup; } @@ -187,7 +187,7 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca) priv->send_cq = ib_create_cq(priv->ca, ipoib_ib_tx_completion, NULL, priv, &cq_attr); if (IS_ERR(priv->send_cq)) { - printk(KERN_WARNING "%s: failed to create send CQ\n", ca->name); + pr_warn("%s: failed to create send CQ\n", ca->name); goto out_free_recv_cq; } @@ -208,7 +208,7 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca) priv->qp = ib_create_qp(priv->pd, &init_attr); if (IS_ERR(priv->qp)) { - printk(KERN_WARNING "%s: failed to create QP\n", ca->name); + pr_warn("%s: failed to create QP\n", ca->name); goto out_free_send_cq; } diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c index 720dfb3a1ac2..fd55163801a3 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.c +++ b/drivers/infiniband/ulp/isert/ib_isert.c @@ -2123,6 +2123,9 @@ isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn, u32 rkey, offset; int ret; + if (cmd->ctx_init_done) + goto rdma_ctx_post; + if (dir == DMA_FROM_DEVICE) { addr = cmd->write_va; rkey = cmd->write_stag; @@ -2150,11 +2153,15 @@ isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn, se_cmd->t_data_sg, se_cmd->t_data_nents, offset, addr, rkey, dir); } + if (ret < 0) { isert_err("Cmd: %p failed to prepare RDMA res\n", cmd); return ret; } + cmd->ctx_init_done = true; + +rdma_ctx_post: ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr); if (ret < 0) isert_err("Cmd: %p failed to post RDMA res\n", cmd); diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h index d6fd248320ae..3b296bac4f60 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.h +++ b/drivers/infiniband/ulp/isert/ib_isert.h @@ -126,6 +126,7 @@ struct isert_cmd { struct rdma_rw_ctx rw; struct work_struct comp_work; struct scatterlist sg; + bool ctx_init_done; }; static inline struct isert_cmd *tx_desc_to_cmd(struct iser_tx_desc *desc) diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c index 972d4b3c5223..62d88212c1b0 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.c +++ b/drivers/infiniband/ulp/srp/ib_srp.c @@ -3110,7 +3110,6 @@ static int srp_parse_options(const char *buf, struct srp_target_port *target) { char *options, *sep_opt; char *p; - char dgid[3]; substring_t args[MAX_OPT_ARGS]; int opt_mask = 0; int token; @@ -3162,16 +3161,10 @@ static int srp_parse_options(const char *buf, struct srp_target_port *target) goto out; } - for (i = 0; i < 16; ++i) { - strlcpy(dgid, p + i * 2, sizeof(dgid)); - if (sscanf(dgid, "%hhx", - &target->orig_dgid.raw[i]) < 1) { - ret = -EINVAL; - kfree(p); - goto out; - } - } + ret = hex2bin(target->orig_dgid.raw, p, 16); kfree(p); + if (ret < 0) + goto out; break; case SRP_OPT_PKEY: |