// SPDX-License-Identifier: GPL-2.0 /* * Shared Memory Communications over RDMA (SMC-R) and RoCE * * Work Requests exploiting Infiniband API * * Work requests (WR) of type ib_post_send or ib_post_recv respectively * are submitted to either RC SQ or RC RQ respectively * (reliably connected send/receive queue) * and become work queue entries (WQEs). * While an SQ WR/WQE is pending, we track it until transmission completion. * Through a send or receive completion queue (CQ) respectively, * we get completion queue entries (CQEs) [aka work completions (WCs)]. * Since the CQ callback is called from IRQ context, we split work by using * bottom halves implemented by tasklets. * * SMC uses this to exchange LLC (link layer control) * and CDC (connection data control) messages. * * Copyright IBM Corp. 2016 * * Author(s): Steffen Maier <maier@linux.vnet.ibm.com> */ #include <linux/atomic.h> #include <linux/hashtable.h> #include <linux/wait.h> #include <rdma/ib_verbs.h> #include <asm/div64.h> #include "smc.h" #include "smc_wr.h" #define SMC_WR_MAX_POLL_CQE 10 /* max. # of compl. queue elements in 1 poll */ #define SMC_WR_RX_HASH_BITS 4 static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS); static DEFINE_SPINLOCK(smc_wr_rx_hash_lock); struct smc_wr_tx_pend { /* control data for a pending send request */ u64 wr_id; /* work request id sent */ smc_wr_tx_handler handler; enum ib_wc_status wc_status; /* CQE status */ struct smc_link *link; u32 idx; struct smc_wr_tx_pend_priv priv; }; /******************************** send queue *********************************/ /*------------------------------- completion --------------------------------*/ static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id) { u32 i; for (i = 0; i < link->wr_tx_cnt; i++) { if (link->wr_tx_pends[i].wr_id == wr_id) return i; } return link->wr_tx_cnt; } static inline void smc_wr_tx_process_cqe(struct ib_wc *wc) { struct smc_wr_tx_pend pnd_snd; struct smc_link *link; u32 pnd_snd_idx; int i; link = wc->qp->qp_context; if (wc->opcode == IB_WC_REG_MR) { if (wc->status) link->wr_reg_state = FAILED; else link->wr_reg_state = CONFIRMED; wake_up(&link->wr_reg_wait); return; } pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id); if (pnd_snd_idx == link->wr_tx_cnt) return; link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status; memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd)); /* clear the full struct smc_wr_tx_pend including .priv */ memset(&link->wr_tx_pends[pnd_snd_idx], 0, sizeof(link->wr_tx_pends[pnd_snd_idx])); memset(&link->wr_tx_bufs[pnd_snd_idx], 0, sizeof(link->wr_tx_bufs[pnd_snd_idx])); if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask)) return; if (wc->status) { struct smc_link_group *lgr; for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { /* clear full struct smc_wr_tx_pend including .priv */ memset(&link->wr_tx_pends[i], 0, sizeof(link->wr_tx_pends[i])); memset(&link->wr_tx_bufs[i], 0, sizeof(link->wr_tx_bufs[i])); clear_bit(i, link->wr_tx_mask); } /* terminate connections of this link group abnormally */ lgr = container_of(link, struct smc_link_group, lnk[SMC_SINGLE_LINK]); smc_lgr_terminate(lgr); } if (pnd_snd.handler) pnd_snd.handler(&pnd_snd.priv, link, wc->status); wake_up(&link->wr_tx_wait); } static void smc_wr_tx_tasklet_fn(unsigned long data) { struct smc_ib_device *dev = (struct smc_ib_device *)data; struct ib_wc wc[SMC_WR_MAX_POLL_CQE]; int i = 0, rc; int polled = 0; again: polled++; do { rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc); if (polled == 1) { ib_req_notify_cq(dev->roce_cq_send, IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); } if (!rc) break; for (i = 0; i < rc; i++) smc_wr_tx_process_cqe(&wc[i]); } while (rc > 0); if (polled == 1) goto again; } void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context) { struct smc_ib_device *dev = (struct smc_ib_device *)cq_context; tasklet_schedule(&dev->send_tasklet); } /*---------------------------- request submission ---------------------------*/ static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx) { *idx = link->wr_tx_cnt; for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) { if (!test_and_set_bit(*idx, link->wr_tx_mask)) return 0; } *idx = link->wr_tx_cnt; return -EBUSY; } /** * smc_wr_tx_get_free_slot() - returns buffer for message assembly, * and sets info for pending transmit tracking * @link: Pointer to smc_link used to later send the message. * @handler: Send completion handler function pointer. * @wr_buf: Out value returns pointer to message buffer. * @wr_pend_priv: Out value returns pointer serving as handler context. * * Return: 0 on success, or -errno on error. */ int smc_wr_tx_get_free_slot(struct smc_link *link, smc_wr_tx_handler handler, struct smc_wr_buf **wr_buf, struct smc_wr_tx_pend_priv **wr_pend_priv) { struct smc_wr_tx_pend *wr_pend; struct ib_send_wr *wr_ib; u64 wr_id; u32 idx; int rc; *wr_buf = NULL; *wr_pend_priv = NULL; if (in_softirq()) { rc = smc_wr_tx_get_free_slot_index(link, &idx); if (rc) return rc; } else { rc = wait_event_interruptible_timeout( link->wr_tx_wait, (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY), SMC_WR_TX_WAIT_FREE_SLOT_TIME); if (!rc) { /* timeout - terminate connections */ struct smc_link_group *lgr; lgr = container_of(link, struct smc_link_group, lnk[SMC_SINGLE_LINK]); smc_lgr_terminate(lgr); return -EPIPE; } if (rc == -ERESTARTSYS) return -EINTR; if (idx == link->wr_tx_cnt) return -EPIPE; } wr_id = smc_wr_tx_get_next_wr_id(link); wr_pend = &link->wr_tx_pends[idx]; wr_pend->wr_id = wr_id; wr_pend->handler = handler; wr_pend->link = link; wr_pend->idx = idx; wr_ib = &link->wr_tx_ibs[idx]; wr_ib->wr_id = wr_id; *wr_buf = &link->wr_tx_bufs[idx]; *wr_pend_priv = &wr_pend->priv; return 0; } int smc_wr_tx_put_slot(struct smc_link *link, struct smc_wr_tx_pend_priv *wr_pend_priv) { struct smc_wr_tx_pend *pend; pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv); if (pend->idx < link->wr_tx_cnt) { /* clear the full struct smc_wr_tx_pend including .priv */ memset(&link->wr_tx_pends[pend->idx], 0, sizeof(link->wr_tx_pends[pend->idx])); memset(&link->wr_tx_bufs[pend->idx], 0, sizeof(link->wr_tx_bufs[pend->idx])); test_and_clear_bit(pend->idx, link->wr_tx_mask); return 1; } return 0; } /* Send prepared WR slot via ib_post_send. * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer */ int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv) { struct ib_send_wr *failed_wr = NULL; struct smc_wr_tx_pend *pend; int rc; ib_req_notify_cq(link->smcibdev->roce_cq_send, IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); pend = container_of(priv, struct smc_wr_tx_pend, priv); rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], &failed_wr); if (rc) smc_wr_tx_put_slot(link, priv); return rc; } /* Register a memory region and wait for result. */ int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr) { struct ib_send_wr *failed_wr = NULL; int rc; ib_req_notify_cq(link->smcibdev->roce_cq_send, IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); link->wr_reg_state = POSTED; link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr; link->wr_reg.mr = mr; link->wr_reg.key = mr->rkey; failed_wr = &link->wr_reg.wr; rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, &failed_wr); WARN_ON(failed_wr != &link->wr_reg.wr); if (rc) return rc; rc = wait_event_interruptible_timeout(link->wr_reg_wait, (link->wr_reg_state != POSTED), SMC_WR_REG_MR_WAIT_TIME); if (!rc) { /* timeout - terminate connections */ struct smc_link_group *lgr; lgr = container_of(link, struct smc_link_group, lnk[SMC_SINGLE_LINK]); smc_lgr_terminate(lgr); return -EPIPE; } if (rc == -ERESTARTSYS) return -EINTR; switch (link->wr_reg_state) { case CONFIRMED: rc = 0; break; case FAILED: rc = -EIO; break; case POSTED: rc = -EPIPE; break; } return rc; } void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_rx_hdr_type, smc_wr_tx_filter filter, smc_wr_tx_dismisser dismisser, unsigned long data) { struct smc_wr_tx_pend_priv *tx_pend; struct smc_wr_rx_hdr *wr_rx; int i; for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[i]; if (wr_rx->type != wr_rx_hdr_type) continue; tx_pend = &link->wr_tx_pends[i].priv; if (filter(tx_pend, data)) dismisser(tx_pend); } } bool smc_wr_tx_has_pending(struct smc_link *link, u8 wr_rx_hdr_type, smc_wr_tx_filter filter, unsigned long data) { struct smc_wr_tx_pend_priv *tx_pend; struct smc_wr_rx_hdr *wr_rx; int i; for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[i]; if (wr_rx->type != wr_rx_hdr_type) continue; tx_pend = &link->wr_tx_pends[i].priv; if (filter(tx_pend, data)) return true; } return false; } /****************************** receive queue ********************************/ int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler) { struct smc_wr_rx_handler *h_iter; int rc = 0; spin_lock(&smc_wr_rx_hash_lock); hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) { if (h_iter->type == handler->type) { rc = -EEXIST; goto out_unlock; } } hash_add(smc_wr_rx_hash, &handler->list, handler->type); out_unlock: spin_unlock(&smc_wr_rx_hash_lock); return rc; } /* Demultiplex a received work request based on the message type to its handler. * Relies on smc_wr_rx_hash having been completely filled before any IB WRs, * and not being modified any more afterwards so we don't need to lock it. */ static inline void smc_wr_rx_demultiplex(struct ib_wc *wc) { struct smc_link *link = (struct smc_link *)wc->qp->qp_context; struct smc_wr_rx_handler *handler; struct smc_wr_rx_hdr *wr_rx; u64 temp_wr_id; u32 index; if (wc->byte_len < sizeof(*wr_rx)) return; /* short message */ temp_wr_id = wc->wr_id; index = do_div(temp_wr_id, link->wr_rx_cnt); wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index]; hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) { if (handler->type == wr_rx->type) handler->handler(wc, wr_rx); } } static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num) { struct smc_link *link; int i; for (i = 0; i < num; i++) { link = wc[i].qp->qp_context; if (wc[i].status == IB_WC_SUCCESS) { smc_wr_rx_demultiplex(&wc[i]); smc_wr_rx_post(link); /* refill WR RX */ } else { struct smc_link_group *lgr; /* handle status errors */ switch (wc[i].status) { case IB_WC_RETRY_EXC_ERR: case IB_WC_RNR_RETRY_EXC_ERR: case IB_WC_WR_FLUSH_ERR: /* terminate connections of this link group * abnormally */ lgr = container_of(link, struct smc_link_group, lnk[SMC_SINGLE_LINK]); smc_lgr_terminate(lgr); break; default: smc_wr_rx_post(link); /* refill WR RX */ break; } } } } static void smc_wr_rx_tasklet_fn(unsigned long data) { struct smc_ib_device *dev = (struct smc_ib_device *)data; struct ib_wc wc[SMC_WR_MAX_POLL_CQE]; int polled = 0; int rc; again: polled++; do { memset(&wc, 0, sizeof(wc)); rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc); if (polled == 1) { ib_req_notify_cq(dev->roce_cq_recv, IB_CQ_SOLICITED_MASK | IB_CQ_REPORT_MISSED_EVENTS); } if (!rc) break; smc_wr_rx_process_cqes(&wc[0], rc); } while (rc > 0); if (polled == 1) goto again; } void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context) { struct smc_ib_device *dev = (struct smc_ib_device *)cq_context; tasklet_schedule(&dev->recv_tasklet); } int smc_wr_rx_post_init(struct smc_link *link) { u32 i; int rc = 0; for (i = 0; i < link->wr_rx_cnt; i++) rc = smc_wr_rx_post(link); return rc; } /***************************** init, exit, misc ******************************/ void smc_wr_remember_qp_attr(struct smc_link *lnk) { struct ib_qp_attr *attr = &lnk->qp_attr; struct ib_qp_init_attr init_attr; memset(attr, 0, sizeof(*attr)); memset(&init_attr, 0, sizeof(init_attr)); ib_query_qp(lnk->roce_qp, attr, IB_QP_STATE | IB_QP_CUR_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_QKEY | IB_QP_AV | IB_QP_PATH_MTU | IB_QP_TIMEOUT | IB_QP_RETRY_CNT | IB_QP_RNR_RETRY | IB_QP_RQ_PSN | IB_QP_ALT_PATH | IB_QP_MIN_RNR_TIMER | IB_QP_SQ_PSN | IB_QP_PATH_MIG_STATE | IB_QP_CAP | IB_QP_DEST_QPN, &init_attr); lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT, lnk->qp_attr.cap.max_send_wr); lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3, lnk->qp_attr.cap.max_recv_wr); } static void smc_wr_init_sge(struct smc_link *lnk) { u32 i; for (i = 0; i < lnk->wr_tx_cnt; i++) { lnk->wr_tx_sges[i].addr = lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE; lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE; lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; lnk->wr_tx_ibs[i].next = NULL; lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i]; lnk->wr_tx_ibs[i].num_sge = 1; lnk->wr_tx_ibs[i].opcode = IB_WR_SEND; lnk->wr_tx_ibs[i].send_flags = IB_SEND_SIGNALED | IB_SEND_SOLICITED; } for (i = 0; i < lnk->wr_rx_cnt; i++) { lnk->wr_rx_sges[i].addr = lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE; lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE; lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; lnk->wr_rx_ibs[i].next = NULL; lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i]; lnk->wr_rx_ibs[i].num_sge = 1; } lnk->wr_reg.wr.next = NULL; lnk->wr_reg.wr.num_sge = 0; lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED; lnk->wr_reg.wr.opcode = IB_WR_REG_MR; lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE; } void smc_wr_free_link(struct smc_link *lnk) { struct ib_device *ibdev; memset(lnk->wr_tx_mask, 0, BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask)); if (!lnk->smcibdev) return; ibdev = lnk->smcibdev->ibdev; if (lnk->wr_rx_dma_addr) { ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, DMA_FROM_DEVICE); lnk->wr_rx_dma_addr = 0; } if (lnk->wr_tx_dma_addr) { ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, DMA_TO_DEVICE); lnk->wr_tx_dma_addr = 0; } } void smc_wr_free_link_mem(struct smc_link *lnk) { kfree(lnk->wr_tx_pends); lnk->wr_tx_pends = NULL; kfree(lnk->wr_tx_mask); lnk->wr_tx_mask = NULL; kfree(lnk->wr_tx_sges); lnk->wr_tx_sges = NULL; kfree(lnk->wr_rx_sges); lnk->wr_rx_sges = NULL; kfree(lnk->wr_rx_ibs); lnk->wr_rx_ibs = NULL; kfree(lnk->wr_tx_ibs); lnk->wr_tx_ibs = NULL; kfree(lnk->wr_tx_bufs); lnk->wr_tx_bufs = NULL; kfree(lnk->wr_rx_bufs); lnk->wr_rx_bufs = NULL; } int smc_wr_alloc_link_mem(struct smc_link *link) { /* allocate link related memory */ link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL); if (!link->wr_tx_bufs) goto no_mem; link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE, GFP_KERNEL); if (!link->wr_rx_bufs) goto no_mem_wr_tx_bufs; link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]), GFP_KERNEL); if (!link->wr_tx_ibs) goto no_mem_wr_rx_bufs; link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3, sizeof(link->wr_rx_ibs[0]), GFP_KERNEL); if (!link->wr_rx_ibs) goto no_mem_wr_tx_ibs; link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]), GFP_KERNEL); if (!link->wr_tx_sges) goto no_mem_wr_rx_ibs; link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3, sizeof(link->wr_rx_sges[0]), GFP_KERNEL); if (!link->wr_rx_sges) goto no_mem_wr_tx_sges; link->wr_tx_mask = kzalloc( BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*link->wr_tx_mask), GFP_KERNEL); if (!link->wr_tx_mask) goto no_mem_wr_rx_sges; link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_pends[0]), GFP_KERNEL); if (!link->wr_tx_pends) goto no_mem_wr_tx_mask; return 0; no_mem_wr_tx_mask: kfree(link->wr_tx_mask); no_mem_wr_rx_sges: kfree(link->wr_rx_sges); no_mem_wr_tx_sges: kfree(link->wr_tx_sges); no_mem_wr_rx_ibs: kfree(link->wr_rx_ibs); no_mem_wr_tx_ibs: kfree(link->wr_tx_ibs); no_mem_wr_rx_bufs: kfree(link->wr_rx_bufs); no_mem_wr_tx_bufs: kfree(link->wr_tx_bufs); no_mem: return -ENOMEM; } void smc_wr_remove_dev(struct smc_ib_device *smcibdev) { tasklet_kill(&smcibdev->recv_tasklet); tasklet_kill(&smcibdev->send_tasklet); } void smc_wr_add_dev(struct smc_ib_device *smcibdev) { tasklet_init(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn, (unsigned long)smcibdev); tasklet_init(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn, (unsigned long)smcibdev); } int smc_wr_create_link(struct smc_link *lnk) { struct ib_device *ibdev = lnk->smcibdev->ibdev; int rc = 0; smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0); lnk->wr_rx_id = 0; lnk->wr_rx_dma_addr = ib_dma_map_single( ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, DMA_FROM_DEVICE); if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) { lnk->wr_rx_dma_addr = 0; rc = -EIO; goto out; } lnk->wr_tx_dma_addr = ib_dma_map_single( ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, DMA_TO_DEVICE); if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) { rc = -EIO; goto dma_unmap; } smc_wr_init_sge(lnk); memset(lnk->wr_tx_mask, 0, BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask)); init_waitqueue_head(&lnk->wr_tx_wait); init_waitqueue_head(&lnk->wr_reg_wait); return rc; dma_unmap: ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, DMA_FROM_DEVICE); lnk->wr_rx_dma_addr = 0; out: return rc; }