diff options
Diffstat (limited to 'drivers/scsi/lpfc/lpfc_scsi.c')
| -rw-r--r-- | drivers/scsi/lpfc/lpfc_scsi.c | 894 |
1 files changed, 299 insertions, 595 deletions
diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c index b4f1a840b3b4..c98f264f1d83 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.c +++ b/drivers/scsi/lpfc/lpfc_scsi.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -83,9 +83,9 @@ lpfc_rport_data_from_scsi_device(struct scsi_device *sdev) } static void -lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb); +lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_io_buf *psb); static void -lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb); +lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_io_buf *psb); static int lpfc_prot_group_type(struct lpfc_hba *phba, struct scsi_cmnd *sc); @@ -180,9 +180,9 @@ lpfc_cmd_guard_csum(struct scsi_cmnd *sc) **/ static void lpfc_sli4_set_rsp_sgl_last(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd) + struct lpfc_io_buf *lpfc_cmd) { - struct sli4_sge *sgl = (struct sli4_sge *)lpfc_cmd->fcp_bpl; + struct sli4_sge *sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl; if (sgl) { sgl += 1; sgl->word2 = le32_to_cpu(sgl->word2); @@ -200,7 +200,7 @@ lpfc_sli4_set_rsp_sgl_last(struct lpfc_hba *phba, * function updates the statistical data for the command completion. **/ static void -lpfc_update_stats(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_update_stats(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { struct lpfc_rport_data *rdata; struct lpfc_nodelist *pnode; @@ -389,12 +389,12 @@ static int lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc) { struct lpfc_hba *phba = vport->phba; - struct lpfc_scsi_buf *psb; + struct lpfc_io_buf *psb; struct ulp_bde64 *bpl; IOCB_t *iocb; dma_addr_t pdma_phys_fcp_cmd; dma_addr_t pdma_phys_fcp_rsp; - dma_addr_t pdma_phys_bpl; + dma_addr_t pdma_phys_sgl; uint16_t iotag; int bcnt, bpl_size; @@ -408,7 +408,7 @@ lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc) (int)sizeof(struct fcp_rsp), bpl_size); for (bcnt = 0; bcnt < num_to_alloc; bcnt++) { - psb = kzalloc(sizeof(struct lpfc_scsi_buf), GFP_KERNEL); + psb = kzalloc(sizeof(struct lpfc_io_buf), GFP_KERNEL); if (!psb) break; @@ -438,14 +438,14 @@ lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc) psb->fcp_cmnd = psb->data; psb->fcp_rsp = psb->data + sizeof(struct fcp_cmnd); - psb->fcp_bpl = psb->data + sizeof(struct fcp_cmnd) + + psb->dma_sgl = psb->data + sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp); /* Initialize local short-hand pointers. */ - bpl = psb->fcp_bpl; + bpl = (struct ulp_bde64 *)psb->dma_sgl; pdma_phys_fcp_cmd = psb->dma_handle; pdma_phys_fcp_rsp = psb->dma_handle + sizeof(struct fcp_cmnd); - pdma_phys_bpl = psb->dma_handle + sizeof(struct fcp_cmnd) + + pdma_phys_sgl = psb->dma_handle + sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp); /* @@ -496,9 +496,9 @@ lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc) iocb->un.fcpi64.bdl.bdeSize = (2 * sizeof(struct ulp_bde64)); iocb->un.fcpi64.bdl.addrLow = - putPaddrLow(pdma_phys_bpl); + putPaddrLow(pdma_phys_sgl); iocb->un.fcpi64.bdl.addrHigh = - putPaddrHigh(pdma_phys_bpl); + putPaddrHigh(pdma_phys_sgl); iocb->ulpBdeCount = 1; iocb->ulpLe = 1; } @@ -506,6 +506,7 @@ lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc) psb->status = IOSTAT_SUCCESS; /* Put it back into the SCSI buffer list */ psb->cur_iocbq.context1 = psb; + spin_lock_init(&psb->buf_lock); lpfc_release_scsi_buf_s3(phba, psb); } @@ -524,20 +525,27 @@ void lpfc_sli4_vport_delete_fcp_xri_aborted(struct lpfc_vport *vport) { struct lpfc_hba *phba = vport->phba; - struct lpfc_scsi_buf *psb, *next_psb; + struct lpfc_io_buf *psb, *next_psb; + struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; + int idx; - if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP)) + if (!(vport->cfg_enable_fc4_type & LPFC_ENABLE_FCP)) return; + spin_lock_irqsave(&phba->hbalock, iflag); - spin_lock(&phba->sli4_hba.abts_scsi_buf_list_lock); - list_for_each_entry_safe(psb, next_psb, - &phba->sli4_hba.lpfc_abts_scsi_buf_list, list) { - if (psb->rdata && psb->rdata->pnode - && psb->rdata->pnode->vport == vport) - psb->rdata = NULL; + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + qp = &phba->sli4_hba.hdwq[idx]; + + spin_lock(&qp->abts_scsi_buf_list_lock); + list_for_each_entry_safe(psb, next_psb, + &qp->lpfc_abts_scsi_buf_list, list) { + if (psb->rdata && psb->rdata->pnode && + psb->rdata->pnode->vport == vport) + psb->rdata = NULL; + } + spin_unlock(&qp->abts_scsi_buf_list_lock); } - spin_unlock(&phba->sli4_hba.abts_scsi_buf_list_lock); spin_unlock_irqrestore(&phba->hbalock, iflag); } @@ -551,11 +559,12 @@ lpfc_sli4_vport_delete_fcp_xri_aborted(struct lpfc_vport *vport) **/ void lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, - struct sli4_wcqe_xri_aborted *axri) + struct sli4_wcqe_xri_aborted *axri, int idx) { uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); uint16_t rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri); - struct lpfc_scsi_buf *psb, *next_psb; + struct lpfc_io_buf *psb, *next_psb; + struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; struct lpfc_iocbq *iocbq; int i; @@ -565,16 +574,19 @@ lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP)) return; + + qp = &phba->sli4_hba.hdwq[idx]; spin_lock_irqsave(&phba->hbalock, iflag); - spin_lock(&phba->sli4_hba.abts_scsi_buf_list_lock); + spin_lock(&qp->abts_scsi_buf_list_lock); list_for_each_entry_safe(psb, next_psb, - &phba->sli4_hba.lpfc_abts_scsi_buf_list, list) { + &qp->lpfc_abts_scsi_buf_list, list) { if (psb->cur_iocbq.sli4_xritag == xri) { list_del(&psb->list); + qp->abts_scsi_io_bufs--; psb->exch_busy = 0; psb->status = IOSTAT_SUCCESS; spin_unlock( - &phba->sli4_hba.abts_scsi_buf_list_lock); + &qp->abts_scsi_buf_list_lock); if (psb->rdata && psb->rdata->pnode) ndlp = psb->rdata->pnode; else @@ -593,7 +605,7 @@ lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, return; } } - spin_unlock(&phba->sli4_hba.abts_scsi_buf_list_lock); + spin_unlock(&qp->abts_scsi_buf_list_lock); for (i = 1; i <= phba->sli.last_iotag; i++) { iocbq = phba->sli.iocbq_lookup[i]; @@ -602,7 +614,7 @@ lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, continue; if (iocbq->sli4_xritag != xri) continue; - psb = container_of(iocbq, struct lpfc_scsi_buf, cur_iocbq); + psb = container_of(iocbq, struct lpfc_io_buf, cur_iocbq); psb->exch_busy = 0; spin_unlock_irqrestore(&phba->hbalock, iflag); if (!list_empty(&pring->txq)) @@ -614,359 +626,6 @@ lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, } /** - * lpfc_sli4_post_scsi_sgl_list - Post blocks of scsi buffer sgls from a list - * @phba: pointer to lpfc hba data structure. - * @post_sblist: pointer to the scsi buffer list. - * - * This routine walks a list of scsi buffers that was passed in. It attempts - * to construct blocks of scsi buffer sgls which contains contiguous xris and - * uses the non-embedded SGL block post mailbox commands to post to the port. - * For single SCSI buffer sgl with non-contiguous xri, if any, it shall use - * embedded SGL post mailbox command for posting. The @post_sblist passed in - * must be local list, thus no lock is needed when manipulate the list. - * - * Returns: 0 = failure, non-zero number of successfully posted buffers. - **/ -static int -lpfc_sli4_post_scsi_sgl_list(struct lpfc_hba *phba, - struct list_head *post_sblist, int sb_count) -{ - struct lpfc_scsi_buf *psb, *psb_next; - int status, sgl_size; - int post_cnt = 0, block_cnt = 0, num_posting = 0, num_posted = 0; - dma_addr_t pdma_phys_bpl1; - int last_xritag = NO_XRI; - LIST_HEAD(prep_sblist); - LIST_HEAD(blck_sblist); - LIST_HEAD(scsi_sblist); - - /* sanity check */ - if (sb_count <= 0) - return -EINVAL; - - sgl_size = phba->cfg_sg_dma_buf_size - - (sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp)); - - list_for_each_entry_safe(psb, psb_next, post_sblist, list) { - list_del_init(&psb->list); - block_cnt++; - if ((last_xritag != NO_XRI) && - (psb->cur_iocbq.sli4_xritag != last_xritag + 1)) { - /* a hole in xri block, form a sgl posting block */ - list_splice_init(&prep_sblist, &blck_sblist); - post_cnt = block_cnt - 1; - /* prepare list for next posting block */ - list_add_tail(&psb->list, &prep_sblist); - block_cnt = 1; - } else { - /* prepare list for next posting block */ - list_add_tail(&psb->list, &prep_sblist); - /* enough sgls for non-embed sgl mbox command */ - if (block_cnt == LPFC_NEMBED_MBOX_SGL_CNT) { - list_splice_init(&prep_sblist, &blck_sblist); - post_cnt = block_cnt; - block_cnt = 0; - } - } - num_posting++; - last_xritag = psb->cur_iocbq.sli4_xritag; - - /* end of repost sgl list condition for SCSI buffers */ - if (num_posting == sb_count) { - if (post_cnt == 0) { - /* last sgl posting block */ - list_splice_init(&prep_sblist, &blck_sblist); - post_cnt = block_cnt; - } else if (block_cnt == 1) { - /* last single sgl with non-contiguous xri */ - if (sgl_size > SGL_PAGE_SIZE) - pdma_phys_bpl1 = psb->dma_phys_bpl + - SGL_PAGE_SIZE; - else - pdma_phys_bpl1 = 0; - status = lpfc_sli4_post_sgl(phba, - psb->dma_phys_bpl, - pdma_phys_bpl1, - psb->cur_iocbq.sli4_xritag); - if (status) { - /* failure, put on abort scsi list */ - psb->exch_busy = 1; - } else { - /* success, put on SCSI buffer list */ - psb->exch_busy = 0; - psb->status = IOSTAT_SUCCESS; - num_posted++; - } - /* success, put on SCSI buffer sgl list */ - list_add_tail(&psb->list, &scsi_sblist); - } - } - - /* continue until a nembed page worth of sgls */ - if (post_cnt == 0) - continue; - - /* post block of SCSI buffer list sgls */ - status = lpfc_sli4_post_scsi_sgl_block(phba, &blck_sblist, - post_cnt); - - /* don't reset xirtag due to hole in xri block */ - if (block_cnt == 0) - last_xritag = NO_XRI; - - /* reset SCSI buffer post count for next round of posting */ - post_cnt = 0; - - /* put posted SCSI buffer-sgl posted on SCSI buffer sgl list */ - while (!list_empty(&blck_sblist)) { - list_remove_head(&blck_sblist, psb, - struct lpfc_scsi_buf, list); - if (status) { - /* failure, put on abort scsi list */ - psb->exch_busy = 1; - } else { - /* success, put on SCSI buffer list */ - psb->exch_busy = 0; - psb->status = IOSTAT_SUCCESS; - num_posted++; - } - list_add_tail(&psb->list, &scsi_sblist); - } - } - /* Push SCSI buffers with sgl posted to the availble list */ - while (!list_empty(&scsi_sblist)) { - list_remove_head(&scsi_sblist, psb, - struct lpfc_scsi_buf, list); - lpfc_release_scsi_buf_s4(phba, psb); - } - return num_posted; -} - -/** - * lpfc_sli4_repost_scsi_sgl_list - Repost all the allocated scsi buffer sgls - * @phba: pointer to lpfc hba data structure. - * - * This routine walks the list of scsi buffers that have been allocated and - * repost them to the port by using SGL block post. This is needed after a - * pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine - * is responsible for moving all scsi buffers on the lpfc_abts_scsi_sgl_list - * to the lpfc_scsi_buf_list. If the repost fails, reject all scsi buffers. - * - * Returns: 0 = success, non-zero failure. - **/ -int -lpfc_sli4_repost_scsi_sgl_list(struct lpfc_hba *phba) -{ - LIST_HEAD(post_sblist); - int num_posted, rc = 0; - - /* get all SCSI buffers need to repost to a local list */ - spin_lock_irq(&phba->scsi_buf_list_get_lock); - spin_lock(&phba->scsi_buf_list_put_lock); - list_splice_init(&phba->lpfc_scsi_buf_list_get, &post_sblist); - list_splice(&phba->lpfc_scsi_buf_list_put, &post_sblist); - spin_unlock(&phba->scsi_buf_list_put_lock); - spin_unlock_irq(&phba->scsi_buf_list_get_lock); - - /* post the list of scsi buffer sgls to port if available */ - if (!list_empty(&post_sblist)) { - num_posted = lpfc_sli4_post_scsi_sgl_list(phba, &post_sblist, - phba->sli4_hba.scsi_xri_cnt); - /* failed to post any scsi buffer, return error */ - if (num_posted == 0) - rc = -EIO; - } - return rc; -} - -/** - * lpfc_new_scsi_buf_s4 - Scsi buffer allocator for HBA with SLI4 IF spec - * @vport: The virtual port for which this call being executed. - * @num_to_allocate: The requested number of buffers to allocate. - * - * This routine allocates scsi buffers for device with SLI-4 interface spec, - * the scsi buffer contains all the necessary information needed to initiate - * a SCSI I/O. After allocating up to @num_to_allocate SCSI buffers and put - * them on a list, it post them to the port by using SGL block post. - * - * Return codes: - * int - number of scsi buffers that were allocated and posted. - * 0 = failure, less than num_to_alloc is a partial failure. - **/ -static int -lpfc_new_scsi_buf_s4(struct lpfc_vport *vport, int num_to_alloc) -{ - struct lpfc_hba *phba = vport->phba; - struct lpfc_scsi_buf *psb; - struct sli4_sge *sgl; - IOCB_t *iocb; - dma_addr_t pdma_phys_fcp_cmd; - dma_addr_t pdma_phys_fcp_rsp; - dma_addr_t pdma_phys_bpl; - uint16_t iotag, lxri = 0; - int bcnt, num_posted, sgl_size; - LIST_HEAD(prep_sblist); - LIST_HEAD(post_sblist); - LIST_HEAD(scsi_sblist); - - sgl_size = phba->cfg_sg_dma_buf_size - - (sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp)); - - lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP, - "9068 ALLOC %d scsi_bufs: %d (%d + %d + %d)\n", - num_to_alloc, phba->cfg_sg_dma_buf_size, sgl_size, - (int)sizeof(struct fcp_cmnd), - (int)sizeof(struct fcp_rsp)); - - for (bcnt = 0; bcnt < num_to_alloc; bcnt++) { - psb = kzalloc(sizeof(struct lpfc_scsi_buf), GFP_KERNEL); - if (!psb) - break; - /* - * Get memory from the pci pool to map the virt space to - * pci bus space for an I/O. The DMA buffer includes space - * for the struct fcp_cmnd, struct fcp_rsp and the number - * of bde's necessary to support the sg_tablesize. - */ - psb->data = dma_pool_zalloc(phba->lpfc_sg_dma_buf_pool, - GFP_KERNEL, &psb->dma_handle); - if (!psb->data) { - kfree(psb); - break; - } - - /* - * 4K Page alignment is CRITICAL to BlockGuard, double check - * to be sure. - */ - if ((phba->sli3_options & LPFC_SLI3_BG_ENABLED) && - (((unsigned long)(psb->data) & - (unsigned long)(SLI4_PAGE_SIZE - 1)) != 0)) { - lpfc_printf_log(phba, KERN_ERR, LOG_FCP, - "3369 Memory alignment error " - "addr=%lx\n", - (unsigned long)psb->data); - dma_pool_free(phba->lpfc_sg_dma_buf_pool, - psb->data, psb->dma_handle); - kfree(psb); - break; - } - - - lxri = lpfc_sli4_next_xritag(phba); - if (lxri == NO_XRI) { - dma_pool_free(phba->lpfc_sg_dma_buf_pool, - psb->data, psb->dma_handle); - kfree(psb); - break; - } - - /* Allocate iotag for psb->cur_iocbq. */ - iotag = lpfc_sli_next_iotag(phba, &psb->cur_iocbq); - if (iotag == 0) { - dma_pool_free(phba->lpfc_sg_dma_buf_pool, - psb->data, psb->dma_handle); - kfree(psb); - lpfc_printf_log(phba, KERN_ERR, LOG_FCP, - "3368 Failed to allocate IOTAG for" - " XRI:0x%x\n", lxri); - lpfc_sli4_free_xri(phba, lxri); - break; - } - psb->cur_iocbq.sli4_lxritag = lxri; - psb->cur_iocbq.sli4_xritag = phba->sli4_hba.xri_ids[lxri]; - psb->cur_iocbq.iocb_flag |= LPFC_IO_FCP; - psb->fcp_bpl = psb->data; - psb->fcp_cmnd = (psb->data + sgl_size); - psb->fcp_rsp = (struct fcp_rsp *)((uint8_t *)psb->fcp_cmnd + - sizeof(struct fcp_cmnd)); - - /* Initialize local short-hand pointers. */ - sgl = (struct sli4_sge *)psb->fcp_bpl; - pdma_phys_bpl = psb->dma_handle; - pdma_phys_fcp_cmd = (psb->dma_handle + sgl_size); - pdma_phys_fcp_rsp = pdma_phys_fcp_cmd + sizeof(struct fcp_cmnd); - - /* - * The first two bdes are the FCP_CMD and FCP_RSP. - * The balance are sg list bdes. Initialize the - * first two and leave the rest for queuecommand. - */ - sgl->addr_hi = cpu_to_le32(putPaddrHigh(pdma_phys_fcp_cmd)); - sgl->addr_lo = cpu_to_le32(putPaddrLow(pdma_phys_fcp_cmd)); - sgl->word2 = le32_to_cpu(sgl->word2); - bf_set(lpfc_sli4_sge_last, sgl, 0); - sgl->word2 = cpu_to_le32(sgl->word2); - sgl->sge_len = cpu_to_le32(sizeof(struct fcp_cmnd)); - sgl++; - - /* Setup the physical region for the FCP RSP */ - sgl->addr_hi = cpu_to_le32(putPaddrHigh(pdma_phys_fcp_rsp)); - sgl->addr_lo = cpu_to_le32(putPaddrLow(pdma_phys_fcp_rsp)); - sgl->word2 = le32_to_cpu(sgl->word2); - bf_set(lpfc_sli4_sge_last, sgl, 1); - sgl->word2 = cpu_to_le32(sgl->word2); - sgl->sge_len = cpu_to_le32(sizeof(struct fcp_rsp)); - - /* - * Since the IOCB for the FCP I/O is built into this - * lpfc_scsi_buf, initialize it with all known data now. - */ - iocb = &psb->cur_iocbq.iocb; - iocb->un.fcpi64.bdl.ulpIoTag32 = 0; - iocb->un.fcpi64.bdl.bdeFlags = BUFF_TYPE_BDE_64; - /* setting the BLP size to 2 * sizeof BDE may not be correct. - * We are setting the bpl to point to out sgl. An sgl's - * entries are 16 bytes, a bpl entries are 12 bytes. - */ - iocb->un.fcpi64.bdl.bdeSize = sizeof(struct fcp_cmnd); - iocb->un.fcpi64.bdl.addrLow = putPaddrLow(pdma_phys_fcp_cmd); - iocb->un.fcpi64.bdl.addrHigh = putPaddrHigh(pdma_phys_fcp_cmd); - iocb->ulpBdeCount = 1; - iocb->ulpLe = 1; - iocb->ulpClass = CLASS3; - psb->cur_iocbq.context1 = psb; - psb->dma_phys_bpl = pdma_phys_bpl; - - /* add the scsi buffer to a post list */ - list_add_tail(&psb->list, &post_sblist); - spin_lock_irq(&phba->scsi_buf_list_get_lock); - phba->sli4_hba.scsi_xri_cnt++; - spin_unlock_irq(&phba->scsi_buf_list_get_lock); - } - lpfc_printf_log(phba, KERN_INFO, LOG_BG | LOG_FCP, - "3021 Allocate %d out of %d requested new SCSI " - "buffers\n", bcnt, num_to_alloc); - - /* post the list of scsi buffer sgls to port if available */ - if (!list_empty(&post_sblist)) - num_posted = lpfc_sli4_post_scsi_sgl_list(phba, - &post_sblist, bcnt); - else - num_posted = 0; - - return num_posted; -} - -/** - * lpfc_new_scsi_buf - Wrapper funciton for scsi buffer allocator - * @vport: The virtual port for which this call being executed. - * @num_to_allocate: The requested number of buffers to allocate. - * - * This routine wraps the actual SCSI buffer allocator function pointer from - * the lpfc_hba struct. - * - * Return codes: - * int - number of scsi buffers that were allocated. - * 0 = failure, less than num_to_alloc is a partial failure. - **/ -static inline int -lpfc_new_scsi_buf(struct lpfc_vport *vport, int num_to_alloc) -{ - return vport->phba->lpfc_new_scsi_buf(vport, num_to_alloc); -} - -/** * lpfc_get_scsi_buf_s3 - Get a scsi buffer from lpfc_scsi_buf_list of the HBA * @phba: The HBA for which this call is being executed. * @@ -977,15 +636,16 @@ lpfc_new_scsi_buf(struct lpfc_vport *vport, int num_to_alloc) * NULL - Error * Pointer to lpfc_scsi_buf - Success **/ -static struct lpfc_scsi_buf* -lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) +static struct lpfc_io_buf * +lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, + struct scsi_cmnd *cmnd) { - struct lpfc_scsi_buf * lpfc_cmd = NULL; + struct lpfc_io_buf *lpfc_cmd = NULL; struct list_head *scsi_buf_list_get = &phba->lpfc_scsi_buf_list_get; unsigned long iflag = 0; spin_lock_irqsave(&phba->scsi_buf_list_get_lock, iflag); - list_remove_head(scsi_buf_list_get, lpfc_cmd, struct lpfc_scsi_buf, + list_remove_head(scsi_buf_list_get, lpfc_cmd, struct lpfc_io_buf, list); if (!lpfc_cmd) { spin_lock(&phba->scsi_buf_list_put_lock); @@ -993,7 +653,7 @@ lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) &phba->lpfc_scsi_buf_list_get); INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_put); list_remove_head(scsi_buf_list_get, lpfc_cmd, - struct lpfc_scsi_buf, list); + struct lpfc_io_buf, list); spin_unlock(&phba->scsi_buf_list_put_lock); } spin_unlock_irqrestore(&phba->scsi_buf_list_get_lock, iflag); @@ -1005,52 +665,107 @@ lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) return lpfc_cmd; } /** - * lpfc_get_scsi_buf_s4 - Get a scsi buffer from lpfc_scsi_buf_list of the HBA + * lpfc_get_scsi_buf_s4 - Get a scsi buffer from io_buf_list of the HBA * @phba: The HBA for which this call is being executed. * - * This routine removes a scsi buffer from head of @phba lpfc_scsi_buf_list list + * This routine removes a scsi buffer from head of @hdwq io_buf_list * and returns to caller. * * Return codes: * NULL - Error * Pointer to lpfc_scsi_buf - Success **/ -static struct lpfc_scsi_buf* -lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) +static struct lpfc_io_buf * +lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, + struct scsi_cmnd *cmnd) { - struct lpfc_scsi_buf *lpfc_cmd, *lpfc_cmd_next; - unsigned long iflag = 0; - int found = 0; + struct lpfc_io_buf *lpfc_cmd; + struct lpfc_sli4_hdw_queue *qp; + struct sli4_sge *sgl; + IOCB_t *iocb; + dma_addr_t pdma_phys_fcp_rsp; + dma_addr_t pdma_phys_fcp_cmd; + uint32_t sgl_size, cpu, idx; + int tag; - spin_lock_irqsave(&phba->scsi_buf_list_get_lock, iflag); - list_for_each_entry_safe(lpfc_cmd, lpfc_cmd_next, - &phba->lpfc_scsi_buf_list_get, list) { - if (lpfc_test_rrq_active(phba, ndlp, - lpfc_cmd->cur_iocbq.sli4_lxritag)) - continue; - list_del_init(&lpfc_cmd->list); - found = 1; - break; - } - if (!found) { - spin_lock(&phba->scsi_buf_list_put_lock); - list_splice(&phba->lpfc_scsi_buf_list_put, - &phba->lpfc_scsi_buf_list_get); - INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_put); - spin_unlock(&phba->scsi_buf_list_put_lock); - list_for_each_entry_safe(lpfc_cmd, lpfc_cmd_next, - &phba->lpfc_scsi_buf_list_get, list) { - if (lpfc_test_rrq_active( - phba, ndlp, lpfc_cmd->cur_iocbq.sli4_lxritag)) - continue; - list_del_init(&lpfc_cmd->list); - found = 1; - break; - } + cpu = smp_processor_id(); + if (cmnd && phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) { + tag = blk_mq_unique_tag(cmnd->request); + idx = blk_mq_unique_tag_to_hwq(tag); + } else { + idx = phba->sli4_hba.cpu_map[cpu].hdwq; } - spin_unlock_irqrestore(&phba->scsi_buf_list_get_lock, iflag); - if (!found) + + lpfc_cmd = lpfc_get_io_buf(phba, ndlp, idx, + !phba->cfg_xri_rebalancing); + if (!lpfc_cmd) { + qp = &phba->sli4_hba.hdwq[idx]; + qp->empty_io_bufs++; return NULL; + } + + sgl_size = phba->cfg_sg_dma_buf_size - + (sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp)); + + /* Setup key fields in buffer that may have been changed + * if other protocols used this buffer. + */ + lpfc_cmd->cur_iocbq.iocb_flag = LPFC_IO_FCP; + lpfc_cmd->prot_seg_cnt = 0; + lpfc_cmd->seg_cnt = 0; + lpfc_cmd->timeout = 0; + lpfc_cmd->flags = 0; + lpfc_cmd->start_time = jiffies; + lpfc_cmd->waitq = NULL; + lpfc_cmd->cpu = cpu; +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + lpfc_cmd->prot_data_type = 0; +#endif + lpfc_cmd->fcp_cmnd = (lpfc_cmd->data + sgl_size); + lpfc_cmd->fcp_rsp = (struct fcp_rsp *)((uint8_t *)lpfc_cmd->fcp_cmnd + + sizeof(struct fcp_cmnd)); + + /* + * The first two SGEs are the FCP_CMD and FCP_RSP. + * The balance are sg list bdes. Initialize the + * first two and leave the rest for queuecommand. + */ + sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl; + pdma_phys_fcp_cmd = (lpfc_cmd->dma_handle + sgl_size); + sgl->addr_hi = cpu_to_le32(putPaddrHigh(pdma_phys_fcp_cmd)); + sgl->addr_lo = cpu_to_le32(putPaddrLow(pdma_phys_fcp_cmd)); + sgl->word2 = le32_to_cpu(sgl->word2); + bf_set(lpfc_sli4_sge_last, sgl, 0); + sgl->word2 = cpu_to_le32(sgl->word2); + sgl->sge_len = cpu_to_le32(sizeof(struct fcp_cmnd)); + sgl++; + + /* Setup the physical region for the FCP RSP */ + pdma_phys_fcp_rsp = pdma_phys_fcp_cmd + sizeof(struct fcp_cmnd); + sgl->addr_hi = cpu_to_le32(putPaddrHigh(pdma_phys_fcp_rsp)); + sgl->addr_lo = cpu_to_le32(putPaddrLow(pdma_phys_fcp_rsp)); + sgl->word2 = le32_to_cpu(sgl->word2); + bf_set(lpfc_sli4_sge_last, sgl, 1); + sgl->word2 = cpu_to_le32(sgl->word2); + sgl->sge_len = cpu_to_le32(sizeof(struct fcp_rsp)); + + /* + * Since the IOCB for the FCP I/O is built into this + * lpfc_io_buf, initialize it with all known data now. + */ + iocb = &lpfc_cmd->cur_iocbq.iocb; + iocb->un.fcpi64.bdl.ulpIoTag32 = 0; + iocb->un.fcpi64.bdl.bdeFlags = BUFF_TYPE_BDE_64; + /* setting the BLP size to 2 * sizeof BDE may not be correct. + * We are setting the bpl to point to out sgl. An sgl's + * entries are 16 bytes, a bpl entries are 12 bytes. + */ + iocb->un.fcpi64.bdl.bdeSize = sizeof(struct fcp_cmnd); + iocb->un.fcpi64.bdl.addrLow = putPaddrLow(pdma_phys_fcp_cmd); + iocb->un.fcpi64.bdl.addrHigh = putPaddrHigh(pdma_phys_fcp_cmd); + iocb->ulpBdeCount = 1; + iocb->ulpLe = 1; + iocb->ulpClass = CLASS3; if (lpfc_ndlp_check_qdepth(phba, ndlp)) { atomic_inc(&ndlp->cmd_pending); @@ -1069,10 +784,11 @@ lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) * NULL - Error * Pointer to lpfc_scsi_buf - Success **/ -static struct lpfc_scsi_buf* -lpfc_get_scsi_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) +static struct lpfc_io_buf* +lpfc_get_scsi_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, + struct scsi_cmnd *cmnd) { - return phba->lpfc_get_scsi_buf(phba, ndlp); + return phba->lpfc_get_scsi_buf(phba, ndlp, cmnd); } /** @@ -1084,12 +800,11 @@ lpfc_get_scsi_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) * lpfc_scsi_buf_list list. **/ static void -lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) +lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_io_buf *psb) { unsigned long iflag = 0; psb->seg_cnt = 0; - psb->nonsg_phys = 0; psb->prot_seg_cnt = 0; spin_lock_irqsave(&phba->scsi_buf_list_put_lock, iflag); @@ -1104,34 +819,29 @@ lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) * @phba: The Hba for which this call is being executed. * @psb: The scsi buffer which is being released. * - * This routine releases @psb scsi buffer by adding it to tail of @phba - * lpfc_scsi_buf_list list. For SLI4 XRI's are tied to the scsi buffer + * This routine releases @psb scsi buffer by adding it to tail of @hdwq + * io_buf_list list. For SLI4 XRI's are tied to the scsi buffer * and cannot be reused for at least RA_TOV amount of time if it was * aborted. **/ static void -lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) +lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_io_buf *psb) { + struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; psb->seg_cnt = 0; - psb->nonsg_phys = 0; psb->prot_seg_cnt = 0; + qp = psb->hdwq; if (psb->exch_busy) { - spin_lock_irqsave(&phba->sli4_hba.abts_scsi_buf_list_lock, - iflag); + spin_lock_irqsave(&qp->abts_scsi_buf_list_lock, iflag); psb->pCmd = NULL; - list_add_tail(&psb->list, - &phba->sli4_hba.lpfc_abts_scsi_buf_list); - spin_unlock_irqrestore(&phba->sli4_hba.abts_scsi_buf_list_lock, - iflag); + list_add_tail(&psb->list, &qp->lpfc_abts_scsi_buf_list); + qp->abts_scsi_io_bufs++; + spin_unlock_irqrestore(&qp->abts_scsi_buf_list_lock, iflag); } else { - psb->pCmd = NULL; - psb->cur_iocbq.iocb_flag = LPFC_IO_FCP; - spin_lock_irqsave(&phba->scsi_buf_list_put_lock, iflag); - list_add_tail(&psb->list, &phba->lpfc_scsi_buf_list_put); - spin_unlock_irqrestore(&phba->scsi_buf_list_put_lock, iflag); + lpfc_release_io_buf(phba, (struct lpfc_io_buf *)psb, qp); } } @@ -1144,7 +854,7 @@ lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) * lpfc_scsi_buf_list list. **/ static void -lpfc_release_scsi_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) +lpfc_release_scsi_buf(struct lpfc_hba *phba, struct lpfc_io_buf *psb) { if ((psb->flags & LPFC_SBUF_BUMP_QDEPTH) && psb->ndlp) atomic_dec(&psb->ndlp->cmd_pending); @@ -1168,12 +878,12 @@ lpfc_release_scsi_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) * 0 - Success **/ static int -lpfc_scsi_prep_dma_buf_s3(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_scsi_prep_dma_buf_s3(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd; struct scatterlist *sgel = NULL; struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd; - struct ulp_bde64 *bpl = lpfc_cmd->fcp_bpl; + struct ulp_bde64 *bpl = (struct ulp_bde64 *)lpfc_cmd->dma_sgl; struct lpfc_iocbq *iocbq = &lpfc_cmd->cur_iocbq; IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb; struct ulp_bde64 *data_bde = iocb_cmd->unsli3.fcp_ext.dbde; @@ -1320,7 +1030,7 @@ lpfc_bg_err_inject(struct lpfc_hba *phba, struct scsi_cmnd *sc, uint32_t *reftag, uint16_t *apptag, uint32_t new_guard) { struct scatterlist *sgpe; /* s/g prot entry */ - struct lpfc_scsi_buf *lpfc_cmd = NULL; + struct lpfc_io_buf *lpfc_cmd = NULL; struct scsi_dif_tuple *src = NULL; struct lpfc_nodelist *ndlp; struct lpfc_rport_data *rdata; @@ -1379,7 +1089,7 @@ lpfc_bg_err_inject(struct lpfc_hba *phba, struct scsi_cmnd *sc, if (sgpe) { src = (struct scsi_dif_tuple *)sg_virt(sgpe); src += blockoff; - lpfc_cmd = (struct lpfc_scsi_buf *)sc->host_scribble; + lpfc_cmd = (struct lpfc_io_buf *)sc->host_scribble; } /* Should we change the Reference Tag */ @@ -2684,7 +2394,7 @@ lpfc_prot_group_type(struct lpfc_hba *phba, struct scsi_cmnd *sc) **/ static int lpfc_bg_scsi_adjust_dl(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd) + struct lpfc_io_buf *lpfc_cmd) { struct scsi_cmnd *sc = lpfc_cmd->pCmd; int fcpdl; @@ -2724,11 +2434,11 @@ lpfc_bg_scsi_adjust_dl(struct lpfc_hba *phba, **/ static int lpfc_bg_scsi_prep_dma_buf_s3(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd) + struct lpfc_io_buf *lpfc_cmd) { struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd; struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd; - struct ulp_bde64 *bpl = lpfc_cmd->fcp_bpl; + struct ulp_bde64 *bpl = (struct ulp_bde64 *)lpfc_cmd->dma_sgl; IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb; uint32_t num_bde = 0; int datasegcnt, protsegcnt, datadir = scsi_cmnd->sc_data_direction; @@ -2904,7 +2614,7 @@ lpfc_bg_csum(uint8_t *data, int count) * what type of T10-DIF error occurred. */ static void -lpfc_calc_bg_err(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_calc_bg_err(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { struct scatterlist *sgpe; /* s/g prot entry */ struct scatterlist *sgde; /* s/g data entry */ @@ -3089,8 +2799,8 @@ out: * -1 - Internal error (bad profile, ...etc) */ static int -lpfc_parse_bg_err(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd, - struct lpfc_iocbq *pIocbOut) +lpfc_parse_bg_err(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd, + struct lpfc_iocbq *pIocbOut) { struct scsi_cmnd *cmd = lpfc_cmd->pCmd; struct sli3_bg_fields *bgf = &pIocbOut->iocb.unsli3.sli3_bg; @@ -3256,12 +2966,12 @@ out: * 0 - Success **/ static int -lpfc_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd; struct scatterlist *sgel = NULL; struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd; - struct sli4_sge *sgl = (struct sli4_sge *)lpfc_cmd->fcp_bpl; + struct sli4_sge *sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl; struct sli4_sge *first_data_sgl; IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb; dma_addr_t physaddr; @@ -3388,6 +3098,7 @@ lpfc_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) lpfc_cmd->cur_iocbq.priority = ((struct lpfc_device_data *) scsi_cmnd->device->hostdata)->priority; } + return 0; } @@ -3402,11 +3113,11 @@ lpfc_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) **/ static int lpfc_bg_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd) + struct lpfc_io_buf *lpfc_cmd) { struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd; struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd; - struct sli4_sge *sgl = (struct sli4_sge *)(lpfc_cmd->fcp_bpl); + struct sli4_sge *sgl = (struct sli4_sge *)(lpfc_cmd->dma_sgl); IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb; uint32_t num_sge = 0; int datasegcnt, protsegcnt, datadir = scsi_cmnd->sc_data_direction; @@ -3578,7 +3289,7 @@ err: * 0 - Success **/ static inline int -lpfc_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { return phba->lpfc_scsi_prep_dma_buf(phba, lpfc_cmd); } @@ -3597,7 +3308,7 @@ lpfc_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) * 0 - Success **/ static inline int -lpfc_bg_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_bg_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { return phba->lpfc_bg_scsi_prep_dma_buf(phba, lpfc_cmd); } @@ -3614,7 +3325,7 @@ lpfc_bg_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) **/ static void lpfc_send_scsi_error_event(struct lpfc_hba *phba, struct lpfc_vport *vport, - struct lpfc_scsi_buf *lpfc_cmd, struct lpfc_iocbq *rsp_iocb) { + struct lpfc_io_buf *lpfc_cmd, struct lpfc_iocbq *rsp_iocb) { struct scsi_cmnd *cmnd = lpfc_cmd->pCmd; struct fcp_rsp *fcprsp = lpfc_cmd->fcp_rsp; uint32_t resp_info = fcprsp->rspStatus2; @@ -3706,7 +3417,7 @@ lpfc_send_scsi_error_event(struct lpfc_hba *phba, struct lpfc_vport *vport, * field of @lpfc_cmd for device with SLI-3 interface spec. **/ static void -lpfc_scsi_unprep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) +lpfc_scsi_unprep_dma_buf(struct lpfc_hba *phba, struct lpfc_io_buf *psb) { /* * There are only two special cases to consider. (1) the scsi command @@ -3725,7 +3436,7 @@ lpfc_scsi_unprep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) /** * lpfc_handler_fcp_err - FCP response handler * @vport: The virtual port for which this call is being executed. - * @lpfc_cmd: Pointer to lpfc_scsi_buf data structure. + * @lpfc_cmd: Pointer to lpfc_io_buf data structure. * @rsp_iocb: The response IOCB which contains FCP error. * * This routine is called to process response IOCB with status field @@ -3733,7 +3444,7 @@ lpfc_scsi_unprep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) * based upon SCSI and FCP error. **/ static void -lpfc_handle_fcp_err(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, +lpfc_handle_fcp_err(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd, struct lpfc_iocbq *rsp_iocb) { struct lpfc_hba *phba = vport->phba; @@ -3912,49 +3623,6 @@ lpfc_handle_fcp_err(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, } /** - * lpfc_sli4_scmd_to_wqidx_distr - scsi command to SLI4 WQ index distribution - * @phba: Pointer to HBA context object. - * - * This routine performs a roundrobin SCSI command to SLI4 FCP WQ index - * distribution. This is called by __lpfc_sli_issue_iocb_s4() with the hbalock - * held. - * If scsi-mq is enabled, get the default block layer mapping of software queues - * to hardware queues. This information is saved in request tag. - * - * Return: index into SLI4 fast-path FCP queue index. - **/ -int lpfc_sli4_scmd_to_wqidx_distr(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd) -{ - struct scsi_cmnd *cmnd = lpfc_cmd->pCmd; - struct lpfc_vector_map_info *cpup; - int chann, cpu; - uint32_t tag; - uint16_t hwq; - - if (cmnd) { - tag = blk_mq_unique_tag(cmnd->request); - hwq = blk_mq_unique_tag_to_hwq(tag); - - return hwq; - } - - if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_CPU - && phba->cfg_fcp_io_channel > 1) { - cpu = smp_processor_id(); - if (cpu < phba->sli4_hba.num_present_cpu) { - cpup = phba->sli4_hba.cpu_map; - cpup += cpu; - return cpup->channel_id; - } - } - chann = atomic_add_return(1, &phba->fcp_qidx); - chann = chann % phba->cfg_fcp_io_channel; - return chann; -} - - -/** * lpfc_scsi_cmd_iocb_cmpl - Scsi cmnd IOCB completion routine * @phba: The Hba for which this call is being executed. * @pIocbIn: The command IOCBQ for the scsi cmnd. @@ -3968,8 +3636,8 @@ static void lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn, struct lpfc_iocbq *pIocbOut) { - struct lpfc_scsi_buf *lpfc_cmd = - (struct lpfc_scsi_buf *) pIocbIn->context1; + struct lpfc_io_buf *lpfc_cmd = + (struct lpfc_io_buf *) pIocbIn->context1; struct lpfc_vport *vport = pIocbIn->vport; struct lpfc_rport_data *rdata = lpfc_cmd->rdata; struct lpfc_nodelist *pnode = rdata->pnode; @@ -3977,14 +3645,35 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn, unsigned long flags; struct lpfc_fast_path_event *fast_path_evt; struct Scsi_Host *shost; + int idx; uint32_t logit = LOG_FCP; +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + int cpu; +#endif - atomic_inc(&phba->fc4ScsiIoCmpls); + /* Guard against abort handler being called at same time */ + spin_lock(&lpfc_cmd->buf_lock); /* Sanity check on return of outstanding command */ cmd = lpfc_cmd->pCmd; - if (!cmd) + if (!cmd) { + lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, + "2621 IO completion: Not an active IO\n"); + spin_unlock(&lpfc_cmd->buf_lock); return; + } + + idx = lpfc_cmd->cur_iocbq.hba_wqidx; + if (phba->sli4_hba.hdwq) + phba->sli4_hba.hdwq[idx].scsi_cstat.io_cmpls++; + +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + if (phba->cpucheck_on & LPFC_CHECK_SCSI_IO) { + cpu = smp_processor_id(); + if (cpu < LPFC_CHECK_CPU_CNT) + phba->sli4_hba.hdwq[idx].cpucheck_cmpl_io[cpu]++; + } +#endif shost = cmd->device->host; lpfc_cmd->result = (pIocbOut->iocb.un.ulpWord[4] & IOERR_PARAM_MASK); @@ -4178,29 +3867,24 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn, } lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd); - /* If pCmd was set to NULL from abort path, do not call scsi_done */ - if (xchg(&lpfc_cmd->pCmd, NULL) == NULL) { - lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP, - "5688 FCP cmd already NULL, sid: 0x%06x, " - "did: 0x%06x, oxid: 0x%04x\n", - vport->fc_myDID, - (pnode) ? pnode->nlp_DID : 0, - phba->sli_rev == LPFC_SLI_REV4 ? - lpfc_cmd->cur_iocbq.sli4_xritag : 0xffff); - return; - } + lpfc_cmd->pCmd = NULL; + spin_unlock(&lpfc_cmd->buf_lock); /* The sdev is not guaranteed to be valid post scsi_done upcall. */ cmd->scsi_done(cmd); /* - * If there is a thread waiting for command completion + * If there is an abort thread waiting for command completion * wake up the thread. */ - spin_lock_irqsave(shost->host_lock, flags); - if (lpfc_cmd->waitq) - wake_up(lpfc_cmd->waitq); - spin_unlock_irqrestore(shost->host_lock, flags); + spin_lock(&lpfc_cmd->buf_lock); + if (unlikely(lpfc_cmd->cur_iocbq.iocb_flag & LPFC_DRIVER_ABORTED)) { + lpfc_cmd->cur_iocbq.iocb_flag &= ~LPFC_DRIVER_ABORTED; + if (lpfc_cmd->waitq) + wake_up(lpfc_cmd->waitq); + lpfc_cmd->waitq = NULL; + } + spin_unlock(&lpfc_cmd->buf_lock); lpfc_release_scsi_buf(phba, lpfc_cmd); } @@ -4233,7 +3917,7 @@ lpfc_fcpcmd_to_iocb(uint8_t *data, struct fcp_cmnd *fcp_cmnd) * to transfer for device with SLI3 interface spec. **/ static void -lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, +lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd, struct lpfc_nodelist *pnode) { struct lpfc_hba *phba = vport->phba; @@ -4241,7 +3925,9 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd; IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb; struct lpfc_iocbq *piocbq = &(lpfc_cmd->cur_iocbq); + struct lpfc_sli4_hdw_queue *hdwq = NULL; int datadir = scsi_cmnd->sc_data_direction; + int idx; uint8_t *ptr; bool sli4; uint32_t fcpdl; @@ -4267,6 +3953,9 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, sli4 = (phba->sli_rev == LPFC_SLI_REV4); piocbq->iocb.un.fcpi.fcpi_XRdy = 0; + idx = lpfc_cmd->hdwq_no; + if (phba->sli4_hba.hdwq) + hdwq = &phba->sli4_hba.hdwq[idx]; /* * There are three possibilities here - use scatter-gather segment, use @@ -4288,19 +3977,22 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, vport->cfg_first_burst_size; } fcp_cmnd->fcpCntl3 = WRITE_DATA; - atomic_inc(&phba->fc4ScsiOutputRequests); + if (hdwq) + hdwq->scsi_cstat.output_requests++; } else { iocb_cmd->ulpCommand = CMD_FCP_IREAD64_CR; iocb_cmd->ulpPU = PARM_READ_CHECK; fcp_cmnd->fcpCntl3 = READ_DATA; - atomic_inc(&phba->fc4ScsiInputRequests); + if (hdwq) + hdwq->scsi_cstat.input_requests++; } } else { iocb_cmd->ulpCommand = CMD_FCP_ICMND64_CR; iocb_cmd->un.fcpi.fcpi_parm = 0; iocb_cmd->ulpPU = 0; fcp_cmnd->fcpCntl3 = 0; - atomic_inc(&phba->fc4ScsiControlRequests); + if (hdwq) + hdwq->scsi_cstat.control_requests++; } if (phba->sli_rev == 3 && !(phba->sli3_options & LPFC_SLI3_BG_ENABLED)) @@ -4328,7 +4020,7 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, /** * lpfc_scsi_prep_task_mgmt_cmd - Convert SLI3 scsi TM cmd to FCP info unit * @vport: The virtual port for which this call is being executed. - * @lpfc_cmd: Pointer to lpfc_scsi_buf data structure. + * @lpfc_cmd: Pointer to lpfc_io_buf data structure. * @lun: Logical unit number. * @task_mgmt_cmd: SCSI task management command. * @@ -4341,7 +4033,7 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, **/ static int lpfc_scsi_prep_task_mgmt_cmd(struct lpfc_vport *vport, - struct lpfc_scsi_buf *lpfc_cmd, + struct lpfc_io_buf *lpfc_cmd, uint64_t lun, uint8_t task_mgmt_cmd) { @@ -4413,14 +4105,12 @@ lpfc_scsi_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp) switch (dev_grp) { case LPFC_PCI_DEV_LP: - phba->lpfc_new_scsi_buf = lpfc_new_scsi_buf_s3; phba->lpfc_scsi_prep_dma_buf = lpfc_scsi_prep_dma_buf_s3; phba->lpfc_bg_scsi_prep_dma_buf = lpfc_bg_scsi_prep_dma_buf_s3; phba->lpfc_release_scsi_buf = lpfc_release_scsi_buf_s3; phba->lpfc_get_scsi_buf = lpfc_get_scsi_buf_s3; break; case LPFC_PCI_DEV_OC: - phba->lpfc_new_scsi_buf = lpfc_new_scsi_buf_s4; phba->lpfc_scsi_prep_dma_buf = lpfc_scsi_prep_dma_buf_s4; phba->lpfc_bg_scsi_prep_dma_buf = lpfc_bg_scsi_prep_dma_buf_s4; phba->lpfc_release_scsi_buf = lpfc_release_scsi_buf_s4; @@ -4452,8 +4142,8 @@ lpfc_tskmgmt_def_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocbq, struct lpfc_iocbq *rspiocbq) { - struct lpfc_scsi_buf *lpfc_cmd = - (struct lpfc_scsi_buf *) cmdiocbq->context1; + struct lpfc_io_buf *lpfc_cmd = + (struct lpfc_io_buf *) cmdiocbq->context1; if (lpfc_cmd) lpfc_release_scsi_buf(phba, lpfc_cmd); return; @@ -4652,9 +4342,12 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) struct lpfc_hba *phba = vport->phba; struct lpfc_rport_data *rdata; struct lpfc_nodelist *ndlp; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device)); - int err; + int err, idx; +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + int cpu; +#endif rdata = lpfc_rport_data_from_scsi_device(cmnd->device); @@ -4718,7 +4411,7 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) } } - lpfc_cmd = lpfc_get_scsi_buf(phba, ndlp); + lpfc_cmd = lpfc_get_scsi_buf(phba, ndlp, cmnd); if (lpfc_cmd == NULL) { lpfc_rampdown_queue_depth(phba); @@ -4735,8 +4428,6 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) lpfc_cmd->pCmd = cmnd; lpfc_cmd->rdata = rdata; lpfc_cmd->ndlp = ndlp; - lpfc_cmd->timeout = 0; - lpfc_cmd->start_time = jiffies; cmnd->host_scribble = (unsigned char *)lpfc_cmd; if (scsi_get_prot_op(cmnd) != SCSI_PROT_NORMAL) { @@ -4771,6 +4462,16 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) lpfc_scsi_prep_cmnd(vport, lpfc_cmd, ndlp); +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + if (phba->cpucheck_on & LPFC_CHECK_SCSI_IO) { + cpu = smp_processor_id(); + if (cpu < LPFC_CHECK_CPU_CNT) { + struct lpfc_sli4_hdw_queue *hdwq = + &phba->sli4_hba.hdwq[lpfc_cmd->hdwq_no]; + hdwq->cpucheck_xmt_io[cpu]++; + } + } +#endif err = lpfc_sli_issue_iocb(phba, LPFC_FCP_RING, &lpfc_cmd->cur_iocbq, SLI_IOCB_RET_IOCB); if (err) { @@ -4791,16 +4492,6 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) (uint32_t) (cmnd->request->timeout / 1000)); - switch (lpfc_cmd->fcp_cmnd->fcpCntl3) { - case WRITE_DATA: - atomic_dec(&phba->fc4ScsiOutputRequests); - break; - case READ_DATA: - atomic_dec(&phba->fc4ScsiInputRequests); - break; - default: - atomic_dec(&phba->fc4ScsiControlRequests); - } goto out_host_busy_free_buf; } if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) { @@ -4811,10 +4502,26 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) lpfc_poll_rearm_timer(phba); } + if (phba->cfg_xri_rebalancing) + lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_cmd->hdwq_no); + return 0; out_host_busy_free_buf: + idx = lpfc_cmd->hdwq_no; lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd); + if (phba->sli4_hba.hdwq) { + switch (lpfc_cmd->fcp_cmnd->fcpCntl3) { + case WRITE_DATA: + phba->sli4_hba.hdwq[idx].scsi_cstat.output_requests--; + break; + case READ_DATA: + phba->sli4_hba.hdwq[idx].scsi_cstat.input_requests--; + break; + default: + phba->sli4_hba.hdwq[idx].scsi_cstat.control_requests--; + } + } lpfc_release_scsi_buf(phba, lpfc_cmd); out_host_busy: return SCSI_MLQUEUE_HOST_BUSY; @@ -4846,7 +4553,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) struct lpfc_hba *phba = vport->phba; struct lpfc_iocbq *iocb; struct lpfc_iocbq *abtsiocb; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; IOCB_t *cmd, *icmd; int ret = SUCCESS, status = 0; struct lpfc_sli_ring *pring_s4 = NULL; @@ -4858,65 +4565,59 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) if (status != 0 && status != SUCCESS) return status; + lpfc_cmd = (struct lpfc_io_buf *)cmnd->host_scribble; + if (!lpfc_cmd) + return ret; + spin_lock_irqsave(&phba->hbalock, flags); /* driver queued commands are in process of being flushed */ if (phba->hba_flag & HBA_FCP_IOQ_FLUSH) { - spin_unlock_irqrestore(&phba->hbalock, flags); lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP, "3168 SCSI Layer abort requested I/O has been " "flushed by LLD.\n"); - return FAILED; + ret = FAILED; + goto out_unlock; } - lpfc_cmd = (struct lpfc_scsi_buf *)cmnd->host_scribble; - if (!lpfc_cmd || !lpfc_cmd->pCmd) { - spin_unlock_irqrestore(&phba->hbalock, flags); + /* Guard against IO completion being called at same time */ + spin_lock(&lpfc_cmd->buf_lock); + + if (!lpfc_cmd->pCmd) { lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP, "2873 SCSI Layer I/O Abort Request IO CMPL Status " "x%x ID %d LUN %llu\n", SUCCESS, cmnd->device->id, cmnd->device->lun); - return SUCCESS; + goto out_unlock_buf; } iocb = &lpfc_cmd->cur_iocbq; if (phba->sli_rev == LPFC_SLI_REV4) { - if (!(phba->cfg_fof) || - (!(iocb->iocb_flag & LPFC_IO_FOF))) { - pring_s4 = - phba->sli4_hba.fcp_wq[iocb->hba_wqidx]->pring; - } else { - iocb->hba_wqidx = 0; - pring_s4 = phba->sli4_hba.oas_wq->pring; - } + pring_s4 = phba->sli4_hba.hdwq[iocb->hba_wqidx].fcp_wq->pring; if (!pring_s4) { ret = FAILED; - goto out_unlock; + goto out_unlock_buf; } spin_lock(&pring_s4->ring_lock); } /* the command is in process of being cancelled */ if (!(iocb->iocb_flag & LPFC_IO_ON_TXCMPLQ)) { - if (phba->sli_rev == LPFC_SLI_REV4) - spin_unlock(&pring_s4->ring_lock); - spin_unlock_irqrestore(&phba->hbalock, flags); lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP, "3169 SCSI Layer abort requested I/O has been " "cancelled by LLD.\n"); - return FAILED; + ret = FAILED; + goto out_unlock_ring; } /* - * If pCmd field of the corresponding lpfc_scsi_buf structure + * If pCmd field of the corresponding lpfc_io_buf structure * points to a different SCSI command, then the driver has * already completed this command, but the midlayer did not * see the completion before the eh fired. Just return SUCCESS. */ if (lpfc_cmd->pCmd != cmnd) { - if (phba->sli_rev == LPFC_SLI_REV4) - spin_unlock(&pring_s4->ring_lock); lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP, "3170 SCSI Layer abort requested I/O has been " "completed by LLD.\n"); - goto out_unlock; + goto out_unlock_ring; } BUG_ON(iocb->context1 != lpfc_cmd); @@ -4927,6 +4628,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) "3389 SCSI Layer I/O Abort Request is pending\n"); if (phba->sli_rev == LPFC_SLI_REV4) spin_unlock(&pring_s4->ring_lock); + spin_unlock(&lpfc_cmd->buf_lock); spin_unlock_irqrestore(&phba->hbalock, flags); goto wait_for_cmpl; } @@ -4934,9 +4636,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) abtsiocb = __lpfc_sli_get_iocbq(phba); if (abtsiocb == NULL) { ret = FAILED; - if (phba->sli_rev == LPFC_SLI_REV4) - spin_unlock(&pring_s4->ring_lock); - goto out_unlock; + goto out_unlock_ring; } /* Indicate the IO is being aborted by the driver. */ @@ -4986,24 +4686,18 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) /* no longer need the lock after this point */ spin_unlock_irqrestore(&phba->hbalock, flags); - if (ret_val == IOCB_ERROR) { - if (phba->sli_rev == LPFC_SLI_REV4) - spin_lock_irqsave(&pring_s4->ring_lock, flags); - else - spin_lock_irqsave(&phba->hbalock, flags); /* Indicate the IO is not being aborted by the driver. */ iocb->iocb_flag &= ~LPFC_DRIVER_ABORTED; lpfc_cmd->waitq = NULL; - if (phba->sli_rev == LPFC_SLI_REV4) - spin_unlock_irqrestore(&pring_s4->ring_lock, flags); - else - spin_unlock_irqrestore(&phba->hbalock, flags); + spin_unlock(&lpfc_cmd->buf_lock); lpfc_sli_release_iocbq(phba, abtsiocb); ret = FAILED; goto out; } + spin_unlock(&lpfc_cmd->buf_lock); + if (phba->cfg_poll & DISABLE_FCP_RING_INT) lpfc_sli_handle_fast_ring_event(phba, &phba->sli.sli3_ring[LPFC_FCP_RING], HA_R0RE_REQ); @@ -5014,9 +4708,7 @@ wait_for_cmpl: (lpfc_cmd->pCmd != cmnd), msecs_to_jiffies(2*vport->cfg_devloss_tmo*1000)); - spin_lock_irqsave(shost->host_lock, flags); - lpfc_cmd->waitq = NULL; - spin_unlock_irqrestore(shost->host_lock, flags); + spin_lock(&lpfc_cmd->buf_lock); if (lpfc_cmd->pCmd == cmnd) { ret = FAILED; @@ -5027,8 +4719,14 @@ wait_for_cmpl: iocb->sli4_xritag, ret, cmnd->device->id, cmnd->device->lun); } + spin_unlock(&lpfc_cmd->buf_lock); goto out; +out_unlock_ring: + if (phba->sli_rev == LPFC_SLI_REV4) + spin_unlock(&pring_s4->ring_lock); +out_unlock_buf: + spin_unlock(&lpfc_cmd->buf_lock); out_unlock: spin_unlock_irqrestore(&phba->hbalock, flags); out: @@ -5066,7 +4764,7 @@ lpfc_taskmgmt_name(uint8_t task_mgmt_cmd) /** * lpfc_check_fcp_rsp - check the returned fcp_rsp to see if task failed * @vport: The virtual port for which this call is being executed. - * @lpfc_cmd: Pointer to lpfc_scsi_buf data structure. + * @lpfc_cmd: Pointer to lpfc_io_buf data structure. * * This routine checks the FCP RSP INFO to see if the tsk mgmt command succeded * @@ -5075,7 +4773,7 @@ lpfc_taskmgmt_name(uint8_t task_mgmt_cmd) * 0x2002 - Success **/ static int -lpfc_check_fcp_rsp(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_check_fcp_rsp(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd) { struct fcp_rsp *fcprsp = lpfc_cmd->fcp_rsp; uint32_t rsp_info; @@ -5150,7 +4848,7 @@ lpfc_send_taskmgmt(struct lpfc_vport *vport, struct scsi_cmnd *cmnd, uint8_t task_mgmt_cmd) { struct lpfc_hba *phba = vport->phba; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct lpfc_iocbq *iocbq; struct lpfc_iocbq *iocbqrsp; struct lpfc_rport_data *rdata; @@ -5163,7 +4861,7 @@ lpfc_send_taskmgmt(struct lpfc_vport *vport, struct scsi_cmnd *cmnd, return FAILED; pnode = rdata->pnode; - lpfc_cmd = lpfc_get_scsi_buf(phba, pnode); + lpfc_cmd = lpfc_get_scsi_buf(phba, pnode, NULL); if (lpfc_cmd == NULL) return FAILED; lpfc_cmd->timeout = phba->cfg_task_mgmt_tmo; @@ -5671,6 +5369,12 @@ lpfc_slave_alloc(struct scsi_device *sdev) } sdev_cnt = atomic_inc_return(&phba->sdev_cnt); + /* For SLI4, all IO buffers are pre-allocated */ + if (phba->sli_rev == LPFC_SLI_REV4) + return 0; + + /* This code path is now ONLY for SLI3 adapters */ + /* * Populate the cmds_per_lun count scsi_bufs into this host's globally * available list of scsi buffers. Don't allocate more than the @@ -5702,7 +5406,7 @@ lpfc_slave_alloc(struct scsi_device *sdev) (phba->cfg_hba_queue_depth - total)); num_to_alloc = phba->cfg_hba_queue_depth - total; } - num_allocated = lpfc_new_scsi_buf(vport, num_to_alloc); + num_allocated = lpfc_new_scsi_buf_s3(vport, num_to_alloc); if (num_to_alloc != num_allocated) { lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP, "0708 Allocation request of %d " |