From eb571eeade2598635f813b3284d02c13a380301e Mon Sep 17 00:00:00 2001 From: Joe Lawrence Date: Wed, 2 Jul 2014 15:35:16 -0400 Subject: block,scsi: verify return pointer from blk_get_request The blk-core dead queue checks introduce an error scenario to blk_get_request that returns NULL if the request queue has been shutdown. This affects the behavior for __GFP_WAIT callers, who should verify the return value before dereferencing. Signed-off-by: Joe Lawrence Acked-by: Jiri Kosina [for pktdvd] Reviewed-by: Jeff Moyer Signed-off-by: Jens Axboe --- drivers/block/paride/pd.c | 2 ++ drivers/block/pktcdvd.c | 2 ++ drivers/scsi/scsi_error.c | 2 ++ 3 files changed, 6 insertions(+) (limited to 'drivers') diff --git a/drivers/block/paride/pd.c b/drivers/block/paride/pd.c index fea7e76a00de..ca831f741d89 100644 --- a/drivers/block/paride/pd.c +++ b/drivers/block/paride/pd.c @@ -722,6 +722,8 @@ static int pd_special_command(struct pd_unit *disk, int err = 0; rq = blk_get_request(disk->gd->queue, READ, __GFP_WAIT); + if (!rq) + return -ENODEV; rq->cmd_type = REQ_TYPE_SPECIAL; rq->special = func; diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index 758ac442c5b5..7fa8c80e8982 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -704,6 +704,8 @@ static int pkt_generic_packet(struct pktcdvd_device *pd, struct packet_command * rq = blk_get_request(q, (cgc->data_direction == CGC_DATA_WRITE) ? WRITE : READ, __GFP_WAIT); + if (!rq) + return -ENODEV; blk_rq_set_block_pc(rq); if (cgc->buflen) { diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c index 5db8454474ee..4c433bf47a06 100644 --- a/drivers/scsi/scsi_error.c +++ b/drivers/scsi/scsi_error.c @@ -1960,6 +1960,8 @@ static void scsi_eh_lock_door(struct scsi_device *sdev) * request becomes available */ req = blk_get_request(sdev->request_queue, READ, GFP_KERNEL); + if (!req) + return; blk_rq_set_block_pc(req); -- cgit v1.2.3 From a492f075450f3ba87de36e5ffe92a9d0c7af9723 Mon Sep 17 00:00:00 2001 From: Joe Lawrence Date: Thu, 28 Aug 2014 08:15:21 -0600 Subject: block,scsi: fixup blk_get_request dead queue scenarios The blk_get_request function may fail in low-memory conditions or during device removal (even if __GFP_WAIT is set). To distinguish between these errors, modify the blk_get_request call stack to return the appropriate ERR_PTR. Verify that all callers check the return status and consider IS_ERR instead of a simple NULL pointer check. For consistency, make a similar change to the blk_mq_alloc_request leg of blk_get_request. It may fail if the queue is dead, or the caller was unwilling to wait. Signed-off-by: Joe Lawrence Acked-by: Jiri Kosina [for pktdvd] Acked-by: Boaz Harrosh [for osd] Reviewed-by: Jeff Moyer Signed-off-by: Jens Axboe --- block/blk-core.c | 34 ++++++++++++++--------------- block/blk-mq.c | 8 +++++-- block/bsg.c | 8 +++---- block/scsi_ioctl.c | 12 +++++----- drivers/block/paride/pd.c | 4 ++-- drivers/block/pktcdvd.c | 4 ++-- drivers/block/sx8.c | 2 +- drivers/cdrom/cdrom.c | 4 ++-- drivers/ide/ide-park.c | 2 +- drivers/scsi/device_handler/scsi_dh_alua.c | 2 +- drivers/scsi/device_handler/scsi_dh_emc.c | 2 +- drivers/scsi/device_handler/scsi_dh_hp_sw.c | 4 ++-- drivers/scsi/device_handler/scsi_dh_rdac.c | 2 +- drivers/scsi/osd/osd_initiator.c | 4 ++-- drivers/scsi/osst.c | 2 +- drivers/scsi/scsi_error.c | 2 +- drivers/scsi/scsi_lib.c | 2 +- drivers/scsi/sg.c | 4 ++-- drivers/scsi/st.c | 2 +- drivers/target/target_core_pscsi.c | 2 +- 20 files changed, 55 insertions(+), 51 deletions(-) (limited to 'drivers') diff --git a/block/blk-core.c b/block/blk-core.c index c359d72e9d76..93603e6ff479 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -933,9 +933,9 @@ static struct io_context *rq_ioc(struct bio *bio) * Get a free request from @q. This function may fail under memory * pressure or if @q is dead. * - * Must be callled with @q->queue_lock held and, - * Returns %NULL on failure, with @q->queue_lock held. - * Returns !%NULL on success, with @q->queue_lock *not held*. + * Must be called with @q->queue_lock held and, + * Returns ERR_PTR on failure, with @q->queue_lock held. + * Returns request pointer on success, with @q->queue_lock *not held*. */ static struct request *__get_request(struct request_list *rl, int rw_flags, struct bio *bio, gfp_t gfp_mask) @@ -949,7 +949,7 @@ static struct request *__get_request(struct request_list *rl, int rw_flags, int may_queue; if (unlikely(blk_queue_dying(q))) - return NULL; + return ERR_PTR(-ENODEV); may_queue = elv_may_queue(q, rw_flags); if (may_queue == ELV_MQUEUE_NO) @@ -974,7 +974,7 @@ static struct request *__get_request(struct request_list *rl, int rw_flags, * process is not a "batcher", and not * exempted by the IO scheduler */ - return NULL; + return ERR_PTR(-ENOMEM); } } } @@ -992,7 +992,7 @@ static struct request *__get_request(struct request_list *rl, int rw_flags, * allocated with any setting of ->nr_requests */ if (rl->count[is_sync] >= (3 * q->nr_requests / 2)) - return NULL; + return ERR_PTR(-ENOMEM); q->nr_rqs[is_sync]++; rl->count[is_sync]++; @@ -1097,7 +1097,7 @@ fail_alloc: rq_starved: if (unlikely(rl->count[is_sync] == 0)) rl->starved[is_sync] = 1; - return NULL; + return ERR_PTR(-ENOMEM); } /** @@ -1110,9 +1110,9 @@ rq_starved: * Get a free request from @q. If %__GFP_WAIT is set in @gfp_mask, this * function keeps retrying under memory pressure and fails iff @q is dead. * - * Must be callled with @q->queue_lock held and, - * Returns %NULL on failure, with @q->queue_lock held. - * Returns !%NULL on success, with @q->queue_lock *not held*. + * Must be called with @q->queue_lock held and, + * Returns ERR_PTR on failure, with @q->queue_lock held. + * Returns request pointer on success, with @q->queue_lock *not held*. */ static struct request *get_request(struct request_queue *q, int rw_flags, struct bio *bio, gfp_t gfp_mask) @@ -1125,12 +1125,12 @@ static struct request *get_request(struct request_queue *q, int rw_flags, rl = blk_get_rl(q, bio); /* transferred to @rq on success */ retry: rq = __get_request(rl, rw_flags, bio, gfp_mask); - if (rq) + if (!IS_ERR(rq)) return rq; if (!(gfp_mask & __GFP_WAIT) || unlikely(blk_queue_dying(q))) { blk_put_rl(rl); - return NULL; + return rq; } /* wait on @rl and retry */ @@ -1167,7 +1167,7 @@ static struct request *blk_old_get_request(struct request_queue *q, int rw, spin_lock_irq(q->queue_lock); rq = get_request(q, rw, NULL, gfp_mask); - if (!rq) + if (IS_ERR(rq)) spin_unlock_irq(q->queue_lock); /* q->queue_lock is unlocked at this point */ @@ -1219,8 +1219,8 @@ struct request *blk_make_request(struct request_queue *q, struct bio *bio, { struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask); - if (unlikely(!rq)) - return ERR_PTR(-ENOMEM); + if (IS_ERR(rq)) + return rq; blk_rq_set_block_pc(rq); @@ -1615,8 +1615,8 @@ get_rq: * Returns with the queue unlocked. */ req = get_request(q, rw_flags, bio, GFP_NOIO); - if (unlikely(!req)) { - bio_endio(bio, -ENODEV); /* @q is dead */ + if (IS_ERR(req)) { + bio_endio(bio, PTR_ERR(req)); /* @q is dead */ goto out_unlock; } diff --git a/block/blk-mq.c b/block/blk-mq.c index 5189cb1e478a..940aa8a34b70 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -218,9 +218,11 @@ struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp, struct blk_mq_hw_ctx *hctx; struct request *rq; struct blk_mq_alloc_data alloc_data; + int ret; - if (blk_mq_queue_enter(q)) - return NULL; + ret = blk_mq_queue_enter(q); + if (ret) + return ERR_PTR(ret); ctx = blk_mq_get_ctx(q); hctx = q->mq_ops->map_queue(q, ctx->cpu); @@ -240,6 +242,8 @@ struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp, ctx = alloc_data.ctx; } blk_mq_put_ctx(ctx); + if (!rq) + return ERR_PTR(-EWOULDBLOCK); return rq; } EXPORT_SYMBOL(blk_mq_alloc_request); diff --git a/block/bsg.c b/block/bsg.c index ff46addde5d8..73c78fd12cc1 100644 --- a/block/bsg.c +++ b/block/bsg.c @@ -270,8 +270,8 @@ bsg_map_hdr(struct bsg_device *bd, struct sg_io_v4 *hdr, fmode_t has_write_perm, * map scatter-gather elements separately and string them to request */ rq = blk_get_request(q, rw, GFP_KERNEL); - if (!rq) - return ERR_PTR(-ENOMEM); + if (IS_ERR(rq)) + return rq; blk_rq_set_block_pc(rq); ret = blk_fill_sgv4_hdr_rq(q, rq, hdr, bd, has_write_perm); @@ -285,8 +285,8 @@ bsg_map_hdr(struct bsg_device *bd, struct sg_io_v4 *hdr, fmode_t has_write_perm, } next_rq = blk_get_request(q, READ, GFP_KERNEL); - if (!next_rq) { - ret = -ENOMEM; + if (IS_ERR(next_rq)) { + ret = PTR_ERR(next_rq); goto out; } rq->next_rq = next_rq; diff --git a/block/scsi_ioctl.c b/block/scsi_ioctl.c index 29d056782833..a8b0d0208448 100644 --- a/block/scsi_ioctl.c +++ b/block/scsi_ioctl.c @@ -318,8 +318,8 @@ static int sg_io(struct request_queue *q, struct gendisk *bd_disk, at_head = 1; rq = blk_get_request(q, writing ? WRITE : READ, GFP_KERNEL); - if (!rq) - return -ENOMEM; + if (IS_ERR(rq)) + return PTR_ERR(rq); blk_rq_set_block_pc(rq); if (blk_fill_sghdr_rq(q, rq, hdr, mode)) { @@ -448,8 +448,8 @@ int sg_scsi_ioctl(struct request_queue *q, struct gendisk *disk, fmode_t mode, } rq = blk_get_request(q, in_len ? WRITE : READ, __GFP_WAIT); - if (!rq) { - err = -ENODEV; + if (IS_ERR(rq)) { + err = PTR_ERR(rq); goto error_free_buffer; } @@ -539,8 +539,8 @@ static int __blk_send_generic(struct request_queue *q, struct gendisk *bd_disk, int err; rq = blk_get_request(q, WRITE, __GFP_WAIT); - if (!rq) - return -ENODEV; + if (IS_ERR(rq)) + return PTR_ERR(rq); blk_rq_set_block_pc(rq); rq->timeout = BLK_DEFAULT_SG_TIMEOUT; rq->cmd[0] = cmd; diff --git a/drivers/block/paride/pd.c b/drivers/block/paride/pd.c index ca831f741d89..d48715b287e6 100644 --- a/drivers/block/paride/pd.c +++ b/drivers/block/paride/pd.c @@ -722,8 +722,8 @@ static int pd_special_command(struct pd_unit *disk, int err = 0; rq = blk_get_request(disk->gd->queue, READ, __GFP_WAIT); - if (!rq) - return -ENODEV; + if (IS_ERR(rq)) + return PTR_ERR(rq); rq->cmd_type = REQ_TYPE_SPECIAL; rq->special = func; diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index 7fa8c80e8982..09e628dafd9d 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -704,8 +704,8 @@ static int pkt_generic_packet(struct pktcdvd_device *pd, struct packet_command * rq = blk_get_request(q, (cgc->data_direction == CGC_DATA_WRITE) ? WRITE : READ, __GFP_WAIT); - if (!rq) - return -ENODEV; + if (IS_ERR(rq)) + return PTR_ERR(rq); blk_rq_set_block_pc(rq); if (cgc->buflen) { diff --git a/drivers/block/sx8.c b/drivers/block/sx8.c index d5e2d12b9d9e..5d552857de41 100644 --- a/drivers/block/sx8.c +++ b/drivers/block/sx8.c @@ -568,7 +568,7 @@ static struct carm_request *carm_get_special(struct carm_host *host) return NULL; rq = blk_get_request(host->oob_q, WRITE /* bogus */, GFP_KERNEL); - if (!rq) { + if (IS_ERR(rq)) { spin_lock_irqsave(&host->lock, flags); carm_put_request(host, crq); spin_unlock_irqrestore(&host->lock, flags); diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c index 898b84bba28a..5d28a45d2960 100644 --- a/drivers/cdrom/cdrom.c +++ b/drivers/cdrom/cdrom.c @@ -2180,8 +2180,8 @@ static int cdrom_read_cdda_bpc(struct cdrom_device_info *cdi, __u8 __user *ubuf, len = nr * CD_FRAMESIZE_RAW; rq = blk_get_request(q, READ, GFP_KERNEL); - if (!rq) { - ret = -ENOMEM; + if (IS_ERR(rq)) { + ret = PTR_ERR(rq); break; } blk_rq_set_block_pc(rq); diff --git a/drivers/ide/ide-park.c b/drivers/ide/ide-park.c index f41558a0bcd1..ca958604cda2 100644 --- a/drivers/ide/ide-park.c +++ b/drivers/ide/ide-park.c @@ -46,7 +46,7 @@ static void issue_park_cmd(ide_drive_t *drive, unsigned long timeout) * timeout has expired, so power management will be reenabled. */ rq = blk_get_request(q, READ, GFP_NOWAIT); - if (unlikely(!rq)) + if (IS_ERR(rq)) goto out; rq->cmd[0] = REQ_UNPARK_HEADS; diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c index 7bcf67eec921..e99507ed0e3c 100644 --- a/drivers/scsi/device_handler/scsi_dh_alua.c +++ b/drivers/scsi/device_handler/scsi_dh_alua.c @@ -115,7 +115,7 @@ static struct request *get_alua_req(struct scsi_device *sdev, rq = blk_get_request(q, rw, GFP_NOIO); - if (!rq) { + if (IS_ERR(rq)) { sdev_printk(KERN_INFO, sdev, "%s: blk_get_request failed\n", __func__); return NULL; diff --git a/drivers/scsi/device_handler/scsi_dh_emc.c b/drivers/scsi/device_handler/scsi_dh_emc.c index 6f07f7fe3aa1..84765384c47c 100644 --- a/drivers/scsi/device_handler/scsi_dh_emc.c +++ b/drivers/scsi/device_handler/scsi_dh_emc.c @@ -275,7 +275,7 @@ static struct request *get_req(struct scsi_device *sdev, int cmd, rq = blk_get_request(sdev->request_queue, (cmd != INQUIRY) ? WRITE : READ, GFP_NOIO); - if (!rq) { + if (IS_ERR(rq)) { sdev_printk(KERN_INFO, sdev, "get_req: blk_get_request failed"); return NULL; } diff --git a/drivers/scsi/device_handler/scsi_dh_hp_sw.c b/drivers/scsi/device_handler/scsi_dh_hp_sw.c index e9d9fea9e272..4ee2759f5299 100644 --- a/drivers/scsi/device_handler/scsi_dh_hp_sw.c +++ b/drivers/scsi/device_handler/scsi_dh_hp_sw.c @@ -117,7 +117,7 @@ static int hp_sw_tur(struct scsi_device *sdev, struct hp_sw_dh_data *h) retry: req = blk_get_request(sdev->request_queue, WRITE, GFP_NOIO); - if (!req) + if (IS_ERR(req)) return SCSI_DH_RES_TEMP_UNAVAIL; blk_rq_set_block_pc(req); @@ -247,7 +247,7 @@ static int hp_sw_start_stop(struct hp_sw_dh_data *h) struct request *req; req = blk_get_request(h->sdev->request_queue, WRITE, GFP_ATOMIC); - if (!req) + if (IS_ERR(req)) return SCSI_DH_RES_TEMP_UNAVAIL; blk_rq_set_block_pc(req); diff --git a/drivers/scsi/device_handler/scsi_dh_rdac.c b/drivers/scsi/device_handler/scsi_dh_rdac.c index 826069db9848..1b5bc9293e37 100644 --- a/drivers/scsi/device_handler/scsi_dh_rdac.c +++ b/drivers/scsi/device_handler/scsi_dh_rdac.c @@ -274,7 +274,7 @@ static struct request *get_rdac_req(struct scsi_device *sdev, rq = blk_get_request(q, rw, GFP_NOIO); - if (!rq) { + if (IS_ERR(rq)) { sdev_printk(KERN_INFO, sdev, "get_rdac_req: blk_get_request failed.\n"); return NULL; diff --git a/drivers/scsi/osd/osd_initiator.c b/drivers/scsi/osd/osd_initiator.c index 5f4cbf0c4759..fd19fd8468ac 100644 --- a/drivers/scsi/osd/osd_initiator.c +++ b/drivers/scsi/osd/osd_initiator.c @@ -1567,8 +1567,8 @@ static struct request *_make_request(struct request_queue *q, bool has_write, struct request *req; req = blk_get_request(q, has_write ? WRITE : READ, flags); - if (unlikely(!req)) - return ERR_PTR(-ENOMEM); + if (IS_ERR(req)) + return req; blk_rq_set_block_pc(req); return req; diff --git a/drivers/scsi/osst.c b/drivers/scsi/osst.c index 0727ea7cc387..dff37a250d79 100644 --- a/drivers/scsi/osst.c +++ b/drivers/scsi/osst.c @@ -362,7 +362,7 @@ static int osst_execute(struct osst_request *SRpnt, const unsigned char *cmd, int write = (data_direction == DMA_TO_DEVICE); req = blk_get_request(SRpnt->stp->device->request_queue, write, GFP_KERNEL); - if (!req) + if (IS_ERR(req)) return DRIVER_ERROR << 24; blk_rq_set_block_pc(req); diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c index 4c433bf47a06..a2c3d3d255a1 100644 --- a/drivers/scsi/scsi_error.c +++ b/drivers/scsi/scsi_error.c @@ -1960,7 +1960,7 @@ static void scsi_eh_lock_door(struct scsi_device *sdev) * request becomes available */ req = blk_get_request(sdev->request_queue, READ, GFP_KERNEL); - if (!req) + if (IS_ERR(req)) return; blk_rq_set_block_pc(req); diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index ce62e8798cc8..972d0a8adf2e 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -221,7 +221,7 @@ int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd, int ret = DRIVER_ERROR << 24; req = blk_get_request(sdev->request_queue, write, __GFP_WAIT); - if (!req) + if (IS_ERR(req)) return ret; blk_rq_set_block_pc(req); diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c index 01cf88888797..60354449d9ed 100644 --- a/drivers/scsi/sg.c +++ b/drivers/scsi/sg.c @@ -1711,9 +1711,9 @@ sg_start_req(Sg_request *srp, unsigned char *cmd) } rq = blk_get_request(q, rw, GFP_ATOMIC); - if (!rq) { + if (IS_ERR(rq)) { kfree(long_cmdp); - return -ENOMEM; + return PTR_ERR(rq); } blk_rq_set_block_pc(rq); diff --git a/drivers/scsi/st.c b/drivers/scsi/st.c index aff9689de0f7..59db5bfc11db 100644 --- a/drivers/scsi/st.c +++ b/drivers/scsi/st.c @@ -490,7 +490,7 @@ static int st_scsi_execute(struct st_request *SRpnt, const unsigned char *cmd, req = blk_get_request(SRpnt->stp->device->request_queue, write, GFP_KERNEL); - if (!req) + if (IS_ERR(req)) return DRIVER_ERROR << 24; blk_rq_set_block_pc(req); diff --git a/drivers/target/target_core_pscsi.c b/drivers/target/target_core_pscsi.c index 943b1dbe859a..70d9f6dabba0 100644 --- a/drivers/target/target_core_pscsi.c +++ b/drivers/target/target_core_pscsi.c @@ -1050,7 +1050,7 @@ pscsi_execute_cmd(struct se_cmd *cmd) req = blk_get_request(pdv->pdv_sd->request_queue, (data_direction == DMA_TO_DEVICE), GFP_KERNEL); - if (!req) { + if (IS_ERR(req)) { pr_err("PSCSI: blk_get_request() failed\n"); ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; goto fail; -- cgit v1.2.3 From bf57229745f849e500ba69ff91e35bc8160a7373 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 13 Sep 2014 16:40:08 -0700 Subject: blk-mq: remove REQ_END Pass an explicit parameter for the last request in a batch to ->queue_rq instead of using a request flag. Besides being a cleaner and non-stateful interface this is also required for the next patch, which fixes the blk-mq I/O submission code to not start a time too early. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-mq.c | 22 +++++----------------- drivers/block/mtip32xx/mtip32xx.c | 3 ++- drivers/block/null_blk.c | 3 ++- drivers/block/virtio_blk.c | 4 ++-- drivers/scsi/scsi_lib.c | 3 ++- include/linux/blk-mq.h | 2 +- include/linux/blk_types.h | 2 -- 7 files changed, 14 insertions(+), 25 deletions(-) (limited to 'drivers') diff --git a/block/blk-mq.c b/block/blk-mq.c index e743d28620b2..32b4797f4186 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -384,7 +384,7 @@ void blk_mq_complete_request(struct request *rq) } EXPORT_SYMBOL(blk_mq_complete_request); -static void blk_mq_start_request(struct request *rq, bool last) +static void blk_mq_start_request(struct request *rq) { struct request_queue *q = rq->q; @@ -421,16 +421,6 @@ static void blk_mq_start_request(struct request *rq, bool last) */ rq->nr_phys_segments++; } - - /* - * Flag the last request in the series so that drivers know when IO - * should be kicked off, if they don't do it on a per-request basis. - * - * Note: the flag isn't the only condition drivers should do kick off. - * If drive is busy, the last request might not have the bit set. - */ - if (last) - rq->cmd_flags |= REQ_END; } static void __blk_mq_requeue_request(struct request *rq) @@ -440,8 +430,6 @@ static void __blk_mq_requeue_request(struct request *rq) trace_block_rq_requeue(q, rq); clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags); - rq->cmd_flags &= ~REQ_END; - if (q->dma_drain_size && blk_rq_bytes(rq)) rq->nr_phys_segments--; } @@ -755,9 +743,9 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) rq = list_first_entry(&rq_list, struct request, queuelist); list_del_init(&rq->queuelist); - blk_mq_start_request(rq, list_empty(&rq_list)); + blk_mq_start_request(rq); - ret = q->mq_ops->queue_rq(hctx, rq); + ret = q->mq_ops->queue_rq(hctx, rq, list_empty(&rq_list)); switch (ret) { case BLK_MQ_RQ_QUEUE_OK: queued++; @@ -1198,14 +1186,14 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio) int ret; blk_mq_bio_to_request(rq, bio); - blk_mq_start_request(rq, true); + blk_mq_start_request(rq); /* * For OK queue, we are done. For error, kill it. Any other * error (busy), just add it to our list as we previously * would have done */ - ret = q->mq_ops->queue_rq(data.hctx, rq); + ret = q->mq_ops->queue_rq(data.hctx, rq, true); if (ret == BLK_MQ_RQ_QUEUE_OK) goto done; else { diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index 5c8e7fe07745..0e2084f37c67 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -3775,7 +3775,8 @@ static bool mtip_check_unal_depth(struct blk_mq_hw_ctx *hctx, return false; } -static int mtip_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *rq) +static int mtip_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *rq, + bool last) { int ret; diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c index 00d469c7f9f7..c5b7315c2c13 100644 --- a/drivers/block/null_blk.c +++ b/drivers/block/null_blk.c @@ -313,7 +313,8 @@ static void null_request_fn(struct request_queue *q) } } -static int null_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *rq) +static int null_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *rq, + bool last) { struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq); diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 0a581400de0f..13756e016797 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -164,14 +164,14 @@ static void virtblk_done(struct virtqueue *vq) spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); } -static int virtio_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *req) +static int virtio_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *req, + bool last) { struct virtio_blk *vblk = hctx->queue->queuedata; struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); unsigned long flags; unsigned int num; int qid = hctx->queue_num; - const bool last = (req->cmd_flags & REQ_END) != 0; int err; bool notify = false; diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 1f2bae475cb7..f1df41168391 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -1855,7 +1855,8 @@ static void scsi_mq_done(struct scsi_cmnd *cmd) blk_mq_complete_request(cmd->request); } -static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *req) +static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *req, + bool last) { struct request_queue *q = req->q; struct scsi_device *sdev = q->queuedata; diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index a1e31f274fcd..9c4e306a9217 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -77,7 +77,7 @@ struct blk_mq_tag_set { struct list_head tag_list; }; -typedef int (queue_rq_fn)(struct blk_mq_hw_ctx *, struct request *); +typedef int (queue_rq_fn)(struct blk_mq_hw_ctx *, struct request *, bool); typedef struct blk_mq_hw_ctx *(map_queue_fn)(struct request_queue *, const int); typedef int (init_hctx_fn)(struct blk_mq_hw_ctx *, void *, unsigned int); typedef void (exit_hctx_fn)(struct blk_mq_hw_ctx *, unsigned int); diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 66c2167f04a9..bb7d66460e7a 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -188,7 +188,6 @@ enum rq_flag_bits { __REQ_MIXED_MERGE, /* merge of different types, fail separately */ __REQ_KERNEL, /* direct IO to kernel pages */ __REQ_PM, /* runtime pm request */ - __REQ_END, /* last of chain of requests */ __REQ_HASHED, /* on IO scheduler merge hash */ __REQ_MQ_INFLIGHT, /* track inflight for MQ */ __REQ_NR_BITS, /* stops here */ @@ -242,7 +241,6 @@ enum rq_flag_bits { #define REQ_SECURE (1ULL << __REQ_SECURE) #define REQ_KERNEL (1ULL << __REQ_KERNEL) #define REQ_PM (1ULL << __REQ_PM) -#define REQ_END (1ULL << __REQ_END) #define REQ_HASHED (1ULL << __REQ_HASHED) #define REQ_MQ_INFLIGHT (1ULL << __REQ_MQ_INFLIGHT) -- cgit v1.2.3 From e2490073cd7c3d6f6ef6e029a208edd4d38efac4 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 13 Sep 2014 16:40:09 -0700 Subject: blk-mq: call blk_mq_start_request from ->queue_rq When we call blk_mq_start_request from the core blk-mq code before calling into ->queue_rq there is a racy window where the timeout handler can hit before we've fully set up the driver specific part of the command. Move the call to blk_mq_start_request into the driver so the driver can start the request only once it is fully set up. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-mq.c | 13 ++++++------- drivers/block/mtip32xx/mtip32xx.c | 2 ++ drivers/block/null_blk.c | 2 ++ drivers/block/virtio_blk.c | 2 ++ drivers/scsi/scsi_lib.c | 1 + include/linux/blk-mq.h | 1 + 6 files changed, 14 insertions(+), 7 deletions(-) (limited to 'drivers') diff --git a/block/blk-mq.c b/block/blk-mq.c index 32b4797f4186..141f2e06803a 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -384,7 +384,7 @@ void blk_mq_complete_request(struct request *rq) } EXPORT_SYMBOL(blk_mq_complete_request); -static void blk_mq_start_request(struct request *rq) +void blk_mq_start_request(struct request *rq) { struct request_queue *q = rq->q; @@ -422,16 +422,18 @@ static void blk_mq_start_request(struct request *rq) rq->nr_phys_segments++; } } +EXPORT_SYMBOL(blk_mq_start_request); static void __blk_mq_requeue_request(struct request *rq) { struct request_queue *q = rq->q; trace_block_rq_requeue(q, rq); - clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags); - if (q->dma_drain_size && blk_rq_bytes(rq)) - rq->nr_phys_segments--; + if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) { + if (q->dma_drain_size && blk_rq_bytes(rq)) + rq->nr_phys_segments--; + } } void blk_mq_requeue_request(struct request *rq) @@ -743,8 +745,6 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) rq = list_first_entry(&rq_list, struct request, queuelist); list_del_init(&rq->queuelist); - blk_mq_start_request(rq); - ret = q->mq_ops->queue_rq(hctx, rq, list_empty(&rq_list)); switch (ret) { case BLK_MQ_RQ_QUEUE_OK: @@ -1186,7 +1186,6 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio) int ret; blk_mq_bio_to_request(rq, bio); - blk_mq_start_request(rq); /* * For OK queue, we are done. For error, kill it. Any other diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index 0e2084f37c67..4042440a0470 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -3783,6 +3783,8 @@ static int mtip_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *rq, if (unlikely(mtip_check_unal_depth(hctx, rq))) return BLK_MQ_RQ_QUEUE_BUSY; + blk_mq_start_request(rq); + ret = mtip_submit_request(hctx, rq); if (likely(!ret)) return BLK_MQ_RQ_QUEUE_OK; diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c index c5b7315c2c13..332ce20d45da 100644 --- a/drivers/block/null_blk.c +++ b/drivers/block/null_blk.c @@ -321,6 +321,8 @@ static int null_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *rq, cmd->rq = rq; cmd->nq = hctx->driver_data; + blk_mq_start_request(rq); + null_handle_cmd(cmd); return BLK_MQ_RQ_QUEUE_OK; } diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 13756e016797..83816bf6882f 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -205,6 +205,8 @@ static int virtio_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *req, } } + blk_mq_start_request(req); + num = blk_rq_map_sg(hctx->queue, vbr->req, vbr->sg); if (num) { if (rq_data_dir(vbr->req) == WRITE) diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index f1df41168391..2dcd9078de48 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -1890,6 +1890,7 @@ static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *req, scsi_init_cmd_errh(cmd); cmd->scsi_done = scsi_mq_done; + blk_mq_start_request(req); reason = scsi_dispatch_cmd(cmd); if (reason) { scsi_set_blocked(cmd, reason); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 9c4e306a9217..878b6f71da48 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -159,6 +159,7 @@ struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag); struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *, const int ctx_index); struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_tag_set *, unsigned int, int); +void blk_mq_start_request(struct request *rq); void blk_mq_end_io(struct request *rq, int error); void __blk_mq_end_io(struct request *rq, int error); -- cgit v1.2.3 From c8a446ad695ada43a885ec12b38411dbd190a11b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 13 Sep 2014 16:40:10 -0700 Subject: blk-mq: rename blk_mq_end_io to blk_mq_end_request Now that we've changed the driver API on the submission side use the opportunity to fix up the name on the completion side to fit into the general scheme. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-flush.c | 4 ++-- block/blk-mq.c | 16 ++++++++-------- drivers/block/mtip32xx/mtip32xx.c | 4 ++-- drivers/block/null_blk.c | 2 +- drivers/block/virtio_blk.c | 2 +- drivers/scsi/scsi_lib.c | 4 ++-- include/linux/blk-mq.h | 4 ++-- 7 files changed, 18 insertions(+), 18 deletions(-) (limited to 'drivers') diff --git a/block/blk-flush.c b/block/blk-flush.c index 3cb5e9e7108a..698e6926388c 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -202,7 +202,7 @@ static bool blk_flush_complete_seq(struct request *rq, unsigned int seq, list_del_init(&rq->flush.list); blk_flush_restore_request(rq); if (q->mq_ops) - blk_mq_end_io(rq, error); + blk_mq_end_request(rq, error); else __blk_end_request_all(rq, error); break; @@ -378,7 +378,7 @@ void blk_insert_flush(struct request *rq) */ if (!policy) { if (q->mq_ops) - blk_mq_end_io(rq, 0); + blk_mq_end_request(rq, 0); else __blk_end_bidi_request(rq, 0, 0, 0); return; diff --git a/block/blk-mq.c b/block/blk-mq.c index 141f2e06803a..1713686f5c2f 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -300,7 +300,7 @@ void blk_mq_clone_flush_request(struct request *flush_rq, hctx->cmd_size); } -inline void __blk_mq_end_io(struct request *rq, int error) +inline void __blk_mq_end_request(struct request *rq, int error) { blk_account_io_done(rq); @@ -312,15 +312,15 @@ inline void __blk_mq_end_io(struct request *rq, int error) blk_mq_free_request(rq); } } -EXPORT_SYMBOL(__blk_mq_end_io); +EXPORT_SYMBOL(__blk_mq_end_request); -void blk_mq_end_io(struct request *rq, int error) +void blk_mq_end_request(struct request *rq, int error) { if (blk_update_request(rq, error, blk_rq_bytes(rq))) BUG(); - __blk_mq_end_io(rq, error); + __blk_mq_end_request(rq, error); } -EXPORT_SYMBOL(blk_mq_end_io); +EXPORT_SYMBOL(blk_mq_end_request); static void __blk_mq_complete_request_remote(void *data) { @@ -360,7 +360,7 @@ void __blk_mq_complete_request(struct request *rq) struct request_queue *q = rq->q; if (!q->softirq_done_fn) - blk_mq_end_io(rq, rq->errors); + blk_mq_end_request(rq, rq->errors); else blk_mq_ipi_complete_request(rq); } @@ -758,7 +758,7 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) pr_err("blk-mq: bad return on queue: %d\n", ret); case BLK_MQ_RQ_QUEUE_ERROR: rq->errors = -EIO; - blk_mq_end_io(rq, rq->errors); + blk_mq_end_request(rq, rq->errors); break; } @@ -1200,7 +1200,7 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio) if (ret == BLK_MQ_RQ_QUEUE_ERROR) { rq->errors = -EIO; - blk_mq_end_io(rq, rq->errors); + blk_mq_end_request(rq, rq->errors); goto done; } } diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index 4042440a0470..6b7e8d0fba99 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -247,7 +247,7 @@ static void mtip_async_complete(struct mtip_port *port, if (unlikely(cmd->unaligned)) up(&port->cmd_slot_unal); - blk_mq_end_io(rq, status ? -EIO : 0); + blk_mq_end_request(rq, status ? -EIO : 0); } /* @@ -3739,7 +3739,7 @@ static int mtip_submit_request(struct blk_mq_hw_ctx *hctx, struct request *rq) int err; err = mtip_send_trim(dd, blk_rq_pos(rq), blk_rq_sectors(rq)); - blk_mq_end_io(rq, err); + blk_mq_end_request(rq, err); return 0; } diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c index 332ce20d45da..ac50a2931044 100644 --- a/drivers/block/null_blk.c +++ b/drivers/block/null_blk.c @@ -177,7 +177,7 @@ static void end_cmd(struct nullb_cmd *cmd) { switch (queue_mode) { case NULL_Q_MQ: - blk_mq_end_io(cmd->rq, 0); + blk_mq_end_request(cmd->rq, 0); return; case NULL_Q_RQ: INIT_LIST_HEAD(&cmd->rq->queuelist); diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 83816bf6882f..f751fc392ba9 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -135,7 +135,7 @@ static inline void virtblk_request_done(struct request *req) req->errors = (error != 0); } - blk_mq_end_io(req, error); + blk_mq_end_request(req, error); } static void virtblk_done(struct virtqueue *vq) diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 2dcd9078de48..73ce7d27f5c8 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -713,7 +713,7 @@ static bool scsi_end_request(struct request *req, int error, if (req->mq_ctx) { /* - * In the MQ case the command gets freed by __blk_mq_end_io, + * In the MQ case the command gets freed by __blk_mq_end_request, * so we have to do all cleanup that depends on it earlier. * * We also can't kick the queues from irq context, so we @@ -721,7 +721,7 @@ static bool scsi_end_request(struct request *req, int error, */ scsi_mq_uninit_cmd(cmd); - __blk_mq_end_io(req, error); + __blk_mq_end_request(req, error); if (scsi_target(sdev)->single_lun || !list_empty(&sdev->host->starved_list)) diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 878b6f71da48..cb217c16990d 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -160,8 +160,8 @@ struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *, const int ctx_ind struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_tag_set *, unsigned int, int); void blk_mq_start_request(struct request *rq); -void blk_mq_end_io(struct request *rq, int error); -void __blk_mq_end_io(struct request *rq, int error); +void blk_mq_end_request(struct request *rq, int error); +void __blk_mq_end_request(struct request *rq, int error); void blk_mq_requeue_request(struct request *rq); void blk_mq_add_to_requeue_list(struct request *rq, bool at_head); -- cgit v1.2.3 From 0152fb6b57c4fae769ee75ea2ae670f4ff39fba9 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 13 Sep 2014 16:40:13 -0700 Subject: blk-mq: pass a reserved argument to the timeout handler Allow blk-mq to pass an argument to the timeout handler to indicate if we're timing out a reserved or regular command. For many drivers those need to be handled different. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-mq.c | 6 +++--- drivers/scsi/scsi_lib.c | 10 +++++++++- include/linux/blk-mq.h | 3 ++- 3 files changed, 14 insertions(+), 5 deletions(-) (limited to 'drivers') diff --git a/block/blk-mq.c b/block/blk-mq.c index 298d6e360661..d12f1983d493 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -530,7 +530,7 @@ struct blk_mq_timeout_data { unsigned int next_set; }; -static void blk_mq_rq_timed_out(struct request *req) +static void blk_mq_rq_timed_out(struct request *req, bool reserved) { struct blk_mq_ops *ops = req->q->mq_ops; enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER; @@ -548,7 +548,7 @@ static void blk_mq_rq_timed_out(struct request *req) return; if (ops->timeout) - ret = ops->timeout(req); + ret = ops->timeout(req, reserved); switch (ret) { case BLK_EH_HANDLED: @@ -576,7 +576,7 @@ static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, if (time_after_eq(jiffies, rq->deadline)) { if (!blk_mark_rq_complete(rq)) - blk_mq_rq_timed_out(rq); + blk_mq_rq_timed_out(rq, reserved); } else if (!data->next_set || time_after(data->next, rq->deadline)) { data->next = rq->deadline; data->next_set = 1; diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 73ce7d27f5c8..86b1156edb82 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -1932,6 +1932,14 @@ out: return ret; } +static enum blk_eh_timer_return scsi_timeout(struct request *req, + bool reserved) +{ + if (reserved) + return BLK_EH_RESET_TIMER; + return scsi_times_out(req); +} + static int scsi_init_request(void *data, struct request *rq, unsigned int hctx_idx, unsigned int request_idx, unsigned int numa_node) @@ -2043,7 +2051,7 @@ static struct blk_mq_ops scsi_mq_ops = { .map_queue = blk_mq_map_queue, .queue_rq = scsi_queue_rq, .complete = scsi_softirq_done, - .timeout = scsi_times_out, + .timeout = scsi_timeout, .init_request = scsi_init_request, .exit_request = scsi_exit_request, }; diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 0eb0f642be4b..325349559fb0 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -79,6 +79,7 @@ struct blk_mq_tag_set { typedef int (queue_rq_fn)(struct blk_mq_hw_ctx *, struct request *, bool); typedef struct blk_mq_hw_ctx *(map_queue_fn)(struct request_queue *, const int); +typedef enum blk_eh_timer_return (timeout_fn)(struct request *, bool); typedef int (init_hctx_fn)(struct blk_mq_hw_ctx *, void *, unsigned int); typedef void (exit_hctx_fn)(struct blk_mq_hw_ctx *, unsigned int); typedef int (init_request_fn)(void *, struct request *, unsigned int, @@ -103,7 +104,7 @@ struct blk_mq_ops { /* * Called on request timeout */ - rq_timed_out_fn *timeout; + timeout_fn *timeout; softirq_done_fn *complete; -- cgit v1.2.3 From fe052529e465daff25225aac769828baa88b7252 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Sep 2014 15:59:31 +0200 Subject: scsi: move blk_mq_start_request call earlier Some ATA drivers need the dma drain size workaround, and thus need to call blk_mq_start_request before the S/G mapping. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/scsi/scsi_lib.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 86b1156edb82..5c5617ec834f 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -1846,6 +1846,8 @@ static int scsi_mq_prep_fn(struct request *req) next_rq->special = bidi_sdb; } + blk_mq_start_request(req); + return scsi_setup_cmnd(sdev, req); } @@ -1880,17 +1882,19 @@ static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *req, if (!scsi_host_queue_ready(q, shost, sdev)) goto out_dec_target_busy; + if (!(req->cmd_flags & REQ_DONTPREP)) { ret = prep_to_mq(scsi_mq_prep_fn(req)); if (ret) goto out_dec_host_busy; req->cmd_flags |= REQ_DONTPREP; + } else { + blk_mq_start_request(req); } scsi_init_cmd_errh(cmd); cmd->scsi_done = scsi_mq_done; - blk_mq_start_request(req); reason = scsi_dispatch_cmd(cmd); if (reason) { scsi_set_blocked(cmd, reason); -- cgit v1.2.3 From 180b2f95dd331010a9930a65c8a18d6d81b94dc1 Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:19:56 -0400 Subject: block: Replace bi_integrity with bi_special For commands like REQ_COPY we need a way to pass extra information along with each bio. Like integrity metadata this information must be available at the bottom of the stack so bi_private does not suffice. Rename the existing bi_integrity field to bi_special and make it a union so we can have different bio extensions for each class of command. We previously used bi_integrity != NULL as a way to identify whether a bio had integrity metadata or not. Introduce a REQ_INTEGRITY to be the indicator now that bi_special can contain different things. In addition, bio_integrity(bio) will now return a pointer to the integrity payload (when applicable). Signed-off-by: Martin K. Petersen Reviewed-by: Christoph Hellwig Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- Documentation/block/data-integrity.txt | 10 +++++----- block/bio-integrity.c | 19 ++++++++++--------- drivers/scsi/sd_dif.c | 8 ++++---- include/linux/bio.h | 11 +++++++++-- include/linux/blk_types.h | 8 ++++++-- include/linux/blkdev.h | 7 ++----- 6 files changed, 36 insertions(+), 27 deletions(-) (limited to 'drivers') diff --git a/Documentation/block/data-integrity.txt b/Documentation/block/data-integrity.txt index b4eacf48053c..4d4de8b09530 100644 --- a/Documentation/block/data-integrity.txt +++ b/Documentation/block/data-integrity.txt @@ -129,11 +129,11 @@ interface for this is being worked on. 4.1 BIO The data integrity patches add a new field to struct bio when -CONFIG_BLK_DEV_INTEGRITY is enabled. bio->bi_integrity is a pointer -to a struct bip which contains the bio integrity payload. Essentially -a bip is a trimmed down struct bio which holds a bio_vec containing -the integrity metadata and the required housekeeping information (bvec -pool, vector count, etc.) +CONFIG_BLK_DEV_INTEGRITY is enabled. bio_integrity(bio) returns a +pointer to a struct bip which contains the bio integrity payload. +Essentially a bip is a trimmed down struct bio which holds a bio_vec +containing the integrity metadata and the required housekeeping +information (bvec pool, vector count, etc.) A kernel subsystem can enable data integrity protection on a bio by calling bio_integrity_alloc(bio). This will allocate and attach the diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 36b788552c3e..bd3125c3c124 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -79,6 +79,7 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, bip->bip_slab = idx; bip->bip_bio = bio; bio->bi_integrity = bip; + bio->bi_rw |= REQ_INTEGRITY; return bip; err: @@ -96,7 +97,7 @@ EXPORT_SYMBOL(bio_integrity_alloc); */ void bio_integrity_free(struct bio *bio) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); struct bio_set *bs = bio->bi_pool; if (bip->bip_owns_buf) @@ -128,7 +129,7 @@ EXPORT_SYMBOL(bio_integrity_free); int bio_integrity_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int offset) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); struct bio_vec *iv; if (bip->bip_vcnt >= bip->bip_max_vcnt) { @@ -229,7 +230,7 @@ EXPORT_SYMBOL(bio_integrity_tag_size); static int bio_integrity_tag(struct bio *bio, void *tag_buf, unsigned int len, int set) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); unsigned int nr_sectors; @@ -304,12 +305,12 @@ static int bio_integrity_generate_verify(struct bio *bio, int operate) struct bio_vec *bv; sector_t sector; unsigned int sectors, ret = 0, i; - void *prot_buf = bio->bi_integrity->bip_buf; + void *prot_buf = bio_integrity(bio)->bip_buf; if (operate) sector = bio->bi_iter.bi_sector; else - sector = bio->bi_integrity->bip_iter.bi_sector; + sector = bio_integrity(bio)->bip_iter.bi_sector; bix.disk_name = bio->bi_bdev->bd_disk->disk_name; bix.sector_size = bi->sector_size; @@ -505,7 +506,7 @@ static void bio_integrity_verify_fn(struct work_struct *work) */ void bio_integrity_endio(struct bio *bio, int error) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); BUG_ON(bip->bip_bio != bio); @@ -536,7 +537,7 @@ EXPORT_SYMBOL(bio_integrity_endio); */ void bio_integrity_advance(struct bio *bio, unsigned int bytes_done) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9); @@ -558,7 +559,7 @@ EXPORT_SYMBOL(bio_integrity_advance); void bio_integrity_trim(struct bio *bio, unsigned int offset, unsigned int sectors) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); bio_integrity_advance(bio, offset << 9); @@ -577,7 +578,7 @@ EXPORT_SYMBOL(bio_integrity_trim); int bio_integrity_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp_mask) { - struct bio_integrity_payload *bip_src = bio_src->bi_integrity; + struct bio_integrity_payload *bip_src = bio_integrity(bio_src); struct bio_integrity_payload *bip; BUG_ON(bip_src == NULL); diff --git a/drivers/scsi/sd_dif.c b/drivers/scsi/sd_dif.c index a7a691d0af7d..29f0477a8708 100644 --- a/drivers/scsi/sd_dif.c +++ b/drivers/scsi/sd_dif.c @@ -383,9 +383,9 @@ void sd_dif_prepare(struct request *rq, sector_t hw_sector, if (bio_flagged(bio, BIO_MAPPED_INTEGRITY)) break; - virt = bio->bi_integrity->bip_iter.bi_sector & 0xffffffff; + virt = bio_integrity(bio)->bip_iter.bi_sector & 0xffffffff; - bip_for_each_vec(iv, bio->bi_integrity, iter) { + bip_for_each_vec(iv, bio_integrity(bio), iter) { sdt = kmap_atomic(iv.bv_page) + iv.bv_offset; @@ -434,9 +434,9 @@ void sd_dif_complete(struct scsi_cmnd *scmd, unsigned int good_bytes) struct bio_vec iv; struct bvec_iter iter; - virt = bio->bi_integrity->bip_iter.bi_sector & 0xffffffff; + virt = bio_integrity(bio)->bip_iter.bi_sector & 0xffffffff; - bip_for_each_vec(iv, bio->bi_integrity, iter) { + bip_for_each_vec(iv, bio_integrity(bio), iter) { sdt = kmap_atomic(iv.bv_page) + iv.bv_offset; diff --git a/include/linux/bio.h b/include/linux/bio.h index 63e399b4fde5..a810a74071b2 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -293,6 +293,15 @@ static inline unsigned bio_segments(struct bio *bio) #define bio_get(bio) atomic_inc(&(bio)->bi_cnt) #if defined(CONFIG_BLK_DEV_INTEGRITY) + +static inline struct bio_integrity_payload *bio_integrity(struct bio *bio) +{ + if (bio->bi_rw & REQ_INTEGRITY) + return bio->bi_integrity; + + return NULL; +} + /* * bio integrity payload */ @@ -661,8 +670,6 @@ struct biovec_slab { for_each_bio(_bio) \ bip_for_each_vec(_bvl, _bio->bi_integrity, _iter) -#define bio_integrity(bio) (bio->bi_integrity != NULL) - extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int); extern void bio_integrity_free(struct bio *); extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int); diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index bb7d66460e7a..6a5d2f2de1b9 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -78,9 +78,11 @@ struct bio { struct io_context *bi_ioc; struct cgroup_subsys_state *bi_css; #endif + union { #if defined(CONFIG_BLK_DEV_INTEGRITY) - struct bio_integrity_payload *bi_integrity; /* data integrity */ + struct bio_integrity_payload *bi_integrity; /* data integrity */ #endif + }; unsigned short bi_vcnt; /* how many bio_vec's */ @@ -162,6 +164,7 @@ enum rq_flag_bits { __REQ_WRITE_SAME, /* write same block many times */ __REQ_NOIDLE, /* don't anticipate more IO after this one */ + __REQ_INTEGRITY, /* I/O includes block integrity payload */ __REQ_FUA, /* forced unit access */ __REQ_FLUSH, /* request for cache flush */ @@ -203,13 +206,14 @@ enum rq_flag_bits { #define REQ_DISCARD (1ULL << __REQ_DISCARD) #define REQ_WRITE_SAME (1ULL << __REQ_WRITE_SAME) #define REQ_NOIDLE (1ULL << __REQ_NOIDLE) +#define REQ_INTEGRITY (1ULL << __REQ_INTEGRITY) #define REQ_FAILFAST_MASK \ (REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT | REQ_FAILFAST_DRIVER) #define REQ_COMMON_MASK \ (REQ_WRITE | REQ_FAILFAST_MASK | REQ_SYNC | REQ_META | REQ_PRIO | \ REQ_DISCARD | REQ_WRITE_SAME | REQ_NOIDLE | REQ_FLUSH | REQ_FUA | \ - REQ_SECURE) + REQ_SECURE | REQ_INTEGRITY) #define REQ_CLONE_MASK REQ_COMMON_MASK #define BIO_NO_ADVANCE_ITER_MASK (REQ_DISCARD|REQ_WRITE_SAME) diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 49f3461e4272..7fcb2caef559 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1514,12 +1514,9 @@ static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk) return disk->integrity; } -static inline int blk_integrity_rq(struct request *rq) +static inline bool blk_integrity_rq(struct request *rq) { - if (rq->bio == NULL) - return 0; - - return bio_integrity(rq->bio); + return rq->cmd_flags & REQ_INTEGRITY; } static inline void blk_queue_max_integrity_segments(struct request_queue *q, -- cgit v1.2.3 From 8492b68bc4025e7bce1d57761bd7c047efda2f81 Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:19:57 -0400 Subject: block: Remove integrity tagging functions None of the filesystems appear interested in using the integrity tagging feature. Potentially because very few storage devices actually permit using the application tag space. Remove the tagging functions. Signed-off-by: Martin K. Petersen Reviewed-by: Christoph Hellwig Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- Documentation/block/data-integrity.txt | 34 ------------ block/bio-integrity.c | 94 +--------------------------------- block/blk-integrity.c | 2 - drivers/scsi/sd_dif.c | 65 ----------------------- include/linux/bio.h | 3 -- include/linux/blkdev.h | 4 -- 6 files changed, 1 insertion(+), 201 deletions(-) (limited to 'drivers') diff --git a/Documentation/block/data-integrity.txt b/Documentation/block/data-integrity.txt index 4d4de8b09530..f56ec97f0d14 100644 --- a/Documentation/block/data-integrity.txt +++ b/Documentation/block/data-integrity.txt @@ -206,36 +206,6 @@ will require extra work due to the application tag. bio_integrity_enabled() returned 1. - int bio_integrity_tag_size(bio); - - If the filesystem wants to use the application tag space it will - first have to find out how much storage space is available. - Because tag space is generally limited (usually 2 bytes per - sector regardless of sector size), the integrity framework - supports interleaving the information between the sectors in an - I/O. - - Filesystems can call bio_integrity_tag_size(bio) to find out how - many bytes of storage are available for that particular bio. - - Another option is bdev_get_tag_size(block_device) which will - return the number of available bytes per hardware sector. - - - int bio_integrity_set_tag(bio, void *tag_buf, len); - - After a successful return from bio_integrity_prep(), - bio_integrity_set_tag() can be used to attach an opaque tag - buffer to a bio. Obviously this only makes sense if the I/O is - a WRITE. - - - int bio_integrity_get_tag(bio, void *tag_buf, len); - - Similarly, at READ I/O completion time the filesystem can - retrieve the tag buffer using bio_integrity_get_tag(). - - 5.3 PASSING EXISTING INTEGRITY METADATA Filesystems that either generate their own integrity metadata or @@ -288,8 +258,6 @@ will require extra work due to the application tag. .name = "STANDARDSBODY-TYPE-VARIANT-CSUM", .generate_fn = my_generate_fn, .verify_fn = my_verify_fn, - .get_tag_fn = my_get_tag_fn, - .set_tag_fn = my_set_tag_fn, .tuple_size = sizeof(struct my_tuple_size), .tag_size = , }; @@ -311,7 +279,5 @@ will require extra work due to the application tag. are available per hardware sector. For DIF this is either 2 or 0 depending on the value of the Control Mode Page ATO bit. - See 6.2 for a description of get_tag_fn and set_tag_fn. - ---------------------------------------------------------------------- 2007-12-24 Martin K. Petersen diff --git a/block/bio-integrity.c b/block/bio-integrity.c index bd3125c3c124..367bb24bb9f1 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -209,90 +209,6 @@ static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi, return bio_integrity_hw_sectors(bi, sectors) * bi->tuple_size; } -/** - * bio_integrity_tag_size - Retrieve integrity tag space - * @bio: bio to inspect - * - * Description: Returns the maximum number of tag bytes that can be - * attached to this bio. Filesystems can use this to determine how - * much metadata to attach to an I/O. - */ -unsigned int bio_integrity_tag_size(struct bio *bio) -{ - struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); - - BUG_ON(bio->bi_iter.bi_size == 0); - - return bi->tag_size * (bio->bi_iter.bi_size / bi->sector_size); -} -EXPORT_SYMBOL(bio_integrity_tag_size); - -static int bio_integrity_tag(struct bio *bio, void *tag_buf, unsigned int len, - int set) -{ - struct bio_integrity_payload *bip = bio_integrity(bio); - struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); - unsigned int nr_sectors; - - BUG_ON(bip->bip_buf == NULL); - - if (bi->tag_size == 0) - return -1; - - nr_sectors = bio_integrity_hw_sectors(bi, - DIV_ROUND_UP(len, bi->tag_size)); - - if (nr_sectors * bi->tuple_size > bip->bip_iter.bi_size) { - printk(KERN_ERR "%s: tag too big for bio: %u > %u\n", __func__, - nr_sectors * bi->tuple_size, bip->bip_iter.bi_size); - return -1; - } - - if (set) - bi->set_tag_fn(bip->bip_buf, tag_buf, nr_sectors); - else - bi->get_tag_fn(bip->bip_buf, tag_buf, nr_sectors); - - return 0; -} - -/** - * bio_integrity_set_tag - Attach a tag buffer to a bio - * @bio: bio to attach buffer to - * @tag_buf: Pointer to a buffer containing tag data - * @len: Length of the included buffer - * - * Description: Use this function to tag a bio by leveraging the extra - * space provided by devices formatted with integrity protection. The - * size of the integrity buffer must be <= to the size reported by - * bio_integrity_tag_size(). - */ -int bio_integrity_set_tag(struct bio *bio, void *tag_buf, unsigned int len) -{ - BUG_ON(bio_data_dir(bio) != WRITE); - - return bio_integrity_tag(bio, tag_buf, len, 1); -} -EXPORT_SYMBOL(bio_integrity_set_tag); - -/** - * bio_integrity_get_tag - Retrieve a tag buffer from a bio - * @bio: bio to retrieve buffer from - * @tag_buf: Pointer to a buffer for the tag data - * @len: Length of the target buffer - * - * Description: Use this function to retrieve the tag buffer from a - * completed I/O. The size of the integrity buffer must be <= to the - * size reported by bio_integrity_tag_size(). - */ -int bio_integrity_get_tag(struct bio *bio, void *tag_buf, unsigned int len) -{ - BUG_ON(bio_data_dir(bio) != READ); - - return bio_integrity_tag(bio, tag_buf, len, 0); -} -EXPORT_SYMBOL(bio_integrity_get_tag); - /** * bio_integrity_generate_verify - Generate/verify integrity metadata for a bio * @bio: bio to generate/verify integrity metadata for @@ -355,14 +271,6 @@ static void bio_integrity_generate(struct bio *bio) bio_integrity_generate_verify(bio, 1); } -static inline unsigned short blk_integrity_tuple_size(struct blk_integrity *bi) -{ - if (bi) - return bi->tuple_size; - - return 0; -} - /** * bio_integrity_prep - Prepare bio for integrity I/O * @bio: bio to prepare @@ -393,7 +301,7 @@ int bio_integrity_prep(struct bio *bio) sectors = bio_integrity_hw_sectors(bi, bio_sectors(bio)); /* Allocate kernel buffer for protection data */ - len = sectors * blk_integrity_tuple_size(bi); + len = sectors * bi->tuple_size; buf = kmalloc(len, GFP_NOIO | q->bounce_gfp); if (unlikely(buf == NULL)) { printk(KERN_ERR "could not allocate integrity buffer\n"); diff --git a/block/blk-integrity.c b/block/blk-integrity.c index 7fbab84399e6..7ac17160ab69 100644 --- a/block/blk-integrity.c +++ b/block/blk-integrity.c @@ -418,8 +418,6 @@ int blk_integrity_register(struct gendisk *disk, struct blk_integrity *template) bi->generate_fn = template->generate_fn; bi->verify_fn = template->verify_fn; bi->tuple_size = template->tuple_size; - bi->set_tag_fn = template->set_tag_fn; - bi->get_tag_fn = template->get_tag_fn; bi->tag_size = template->tag_size; } else bi->name = bi_unsupported_name; diff --git a/drivers/scsi/sd_dif.c b/drivers/scsi/sd_dif.c index 29f0477a8708..38a7778631be 100644 --- a/drivers/scsi/sd_dif.c +++ b/drivers/scsi/sd_dif.c @@ -128,39 +128,10 @@ static int sd_dif_type1_verify_ip(struct blk_integrity_exchg *bix) return sd_dif_type1_verify(bix, sd_dif_ip_fn); } -/* - * Functions for interleaving and deinterleaving application tags - */ -static void sd_dif_type1_set_tag(void *prot, void *tag_buf, unsigned int sectors) -{ - struct sd_dif_tuple *sdt = prot; - u8 *tag = tag_buf; - unsigned int i, j; - - for (i = 0, j = 0 ; i < sectors ; i++, j += 2, sdt++) { - sdt->app_tag = tag[j] << 8 | tag[j+1]; - BUG_ON(sdt->app_tag == 0xffff); - } -} - -static void sd_dif_type1_get_tag(void *prot, void *tag_buf, unsigned int sectors) -{ - struct sd_dif_tuple *sdt = prot; - u8 *tag = tag_buf; - unsigned int i, j; - - for (i = 0, j = 0 ; i < sectors ; i++, j += 2, sdt++) { - tag[j] = (sdt->app_tag & 0xff00) >> 8; - tag[j+1] = sdt->app_tag & 0xff; - } -} - static struct blk_integrity dif_type1_integrity_crc = { .name = "T10-DIF-TYPE1-CRC", .generate_fn = sd_dif_type1_generate_crc, .verify_fn = sd_dif_type1_verify_crc, - .get_tag_fn = sd_dif_type1_get_tag, - .set_tag_fn = sd_dif_type1_set_tag, .tuple_size = sizeof(struct sd_dif_tuple), .tag_size = 0, }; @@ -169,8 +140,6 @@ static struct blk_integrity dif_type1_integrity_ip = { .name = "T10-DIF-TYPE1-IP", .generate_fn = sd_dif_type1_generate_ip, .verify_fn = sd_dif_type1_verify_ip, - .get_tag_fn = sd_dif_type1_get_tag, - .set_tag_fn = sd_dif_type1_set_tag, .tuple_size = sizeof(struct sd_dif_tuple), .tag_size = 0, }; @@ -245,42 +214,10 @@ static int sd_dif_type3_verify_ip(struct blk_integrity_exchg *bix) return sd_dif_type3_verify(bix, sd_dif_ip_fn); } -static void sd_dif_type3_set_tag(void *prot, void *tag_buf, unsigned int sectors) -{ - struct sd_dif_tuple *sdt = prot; - u8 *tag = tag_buf; - unsigned int i, j; - - for (i = 0, j = 0 ; i < sectors ; i++, j += 6, sdt++) { - sdt->app_tag = tag[j] << 8 | tag[j+1]; - sdt->ref_tag = tag[j+2] << 24 | tag[j+3] << 16 | - tag[j+4] << 8 | tag[j+5]; - } -} - -static void sd_dif_type3_get_tag(void *prot, void *tag_buf, unsigned int sectors) -{ - struct sd_dif_tuple *sdt = prot; - u8 *tag = tag_buf; - unsigned int i, j; - - for (i = 0, j = 0 ; i < sectors ; i++, j += 2, sdt++) { - tag[j] = (sdt->app_tag & 0xff00) >> 8; - tag[j+1] = sdt->app_tag & 0xff; - tag[j+2] = (sdt->ref_tag & 0xff000000) >> 24; - tag[j+3] = (sdt->ref_tag & 0xff0000) >> 16; - tag[j+4] = (sdt->ref_tag & 0xff00) >> 8; - tag[j+5] = sdt->ref_tag & 0xff; - BUG_ON(sdt->app_tag == 0xffff || sdt->ref_tag == 0xffffffff); - } -} - static struct blk_integrity dif_type3_integrity_crc = { .name = "T10-DIF-TYPE3-CRC", .generate_fn = sd_dif_type3_generate_crc, .verify_fn = sd_dif_type3_verify_crc, - .get_tag_fn = sd_dif_type3_get_tag, - .set_tag_fn = sd_dif_type3_set_tag, .tuple_size = sizeof(struct sd_dif_tuple), .tag_size = 0, }; @@ -289,8 +226,6 @@ static struct blk_integrity dif_type3_integrity_ip = { .name = "T10-DIF-TYPE3-IP", .generate_fn = sd_dif_type3_generate_ip, .verify_fn = sd_dif_type3_verify_ip, - .get_tag_fn = sd_dif_type3_get_tag, - .set_tag_fn = sd_dif_type3_set_tag, .tuple_size = sizeof(struct sd_dif_tuple), .tag_size = 0, }; diff --git a/include/linux/bio.h b/include/linux/bio.h index a810a74071b2..63a0e53e238c 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -362,7 +362,6 @@ extern struct bio *bio_clone_fast(struct bio *, gfp_t, struct bio_set *); extern struct bio *bio_clone_bioset(struct bio *, gfp_t, struct bio_set *bs); extern struct bio_set *fs_bio_set; -unsigned int bio_integrity_tag_size(struct bio *bio); static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs) { @@ -674,8 +673,6 @@ extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, un extern void bio_integrity_free(struct bio *); extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int); extern bool bio_integrity_enabled(struct bio *bio); -extern int bio_integrity_set_tag(struct bio *, void *, unsigned int); -extern int bio_integrity_get_tag(struct bio *, void *, unsigned int); extern int bio_integrity_prep(struct bio *); extern void bio_integrity_endio(struct bio *, int); extern void bio_integrity_advance(struct bio *, unsigned int); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 7fcb2caef559..0bf5d79d9ba0 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1472,14 +1472,10 @@ struct blk_integrity_exchg { typedef void (integrity_gen_fn) (struct blk_integrity_exchg *); typedef int (integrity_vrfy_fn) (struct blk_integrity_exchg *); -typedef void (integrity_set_tag_fn) (void *, void *, unsigned int); -typedef void (integrity_get_tag_fn) (void *, void *, unsigned int); struct blk_integrity { integrity_gen_fn *generate_fn; integrity_vrfy_fn *verify_fn; - integrity_set_tag_fn *set_tag_fn; - integrity_get_tag_fn *get_tag_fn; unsigned short flags; unsigned short tuple_size; -- cgit v1.2.3 From 3be91c4a3d090bd700bd6ee5bf457c1bbf189a4f Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:19:59 -0400 Subject: block: Deprecate the use of the term sector in the context of block integrity The protection interval is not necessarily tied to the logical block size of a block device. Stop using the terms "sector" and "sectors". Going forward we will use the term "seed" to describe the initial reference tag value for a given I/O. "Interval" will be used to describe the portion of the data buffer that a given piece of protection information is associated with. Signed-off-by: Martin K. Petersen Reviewed-by: Christoph Hellwig Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/bio-integrity.c | 42 +++++++++++++++++++++--------------------- block/blk-integrity.c | 10 +++++----- drivers/scsi/sd_dif.c | 46 +++++++++++++++++++++++----------------------- include/linux/blkdev.h | 6 +++--- 4 files changed, 52 insertions(+), 52 deletions(-) (limited to 'drivers') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index e84f7fb8694b..6a3aacf57b19 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -185,20 +185,20 @@ bool bio_integrity_enabled(struct bio *bio) EXPORT_SYMBOL(bio_integrity_enabled); /** - * bio_integrity_hw_sectors - Convert 512b sectors to hardware ditto + * bio_integrity_intervals - Return number of integrity intervals for a bio * @bi: blk_integrity profile for device - * @sectors: Number of 512 sectors to convert + * @sectors: Size of the bio in 512-byte sectors * * Description: The block layer calculates everything in 512 byte - * sectors but integrity metadata is done in terms of the hardware - * sector size of the storage device. Convert the block layer sectors - * to physical sectors. + * sectors but integrity metadata is done in terms of the data integrity + * interval size of the storage device. Convert the block layer sectors + * to the appropriate number of integrity intervals. */ -static inline unsigned int bio_integrity_hw_sectors(struct blk_integrity *bi, - unsigned int sectors) +static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi, + unsigned int sectors) { /* At this point there are only 512b or 4096b DIF/EPP devices */ - if (bi->sector_size == 4096) + if (bi->interval == 4096) return sectors >>= 3; return sectors; @@ -207,7 +207,7 @@ static inline unsigned int bio_integrity_hw_sectors(struct blk_integrity *bi, static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi, unsigned int sectors) { - return bio_integrity_hw_sectors(bi, sectors) * bi->tuple_size; + return bio_integrity_intervals(bi, sectors) * bi->tuple_size; } /** @@ -221,25 +221,25 @@ static int bio_integrity_generate_verify(struct bio *bio, int operate) struct blk_integrity_exchg bix; struct bio_vec *bv; struct bio_integrity_payload *bip = bio_integrity(bio); - sector_t sector; - unsigned int sectors, ret = 0, i; + sector_t seed; + unsigned int intervals, ret = 0, i; void *prot_buf = page_address(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset; if (operate) - sector = bio->bi_iter.bi_sector; + seed = bio->bi_iter.bi_sector; else - sector = bip->bip_iter.bi_sector; + seed = bip->bip_iter.bi_sector; bix.disk_name = bio->bi_bdev->bd_disk->disk_name; - bix.sector_size = bi->sector_size; + bix.interval = bi->interval; bio_for_each_segment_all(bv, bio, i) { void *kaddr = kmap_atomic(bv->bv_page); bix.data_buf = kaddr + bv->bv_offset; bix.data_size = bv->bv_len; bix.prot_buf = prot_buf; - bix.sector = sector; + bix.seed = seed; if (operate) bi->generate_fn(&bix); @@ -251,9 +251,9 @@ static int bio_integrity_generate_verify(struct bio *bio, int operate) } } - sectors = bv->bv_len / bi->sector_size; - sector += sectors; - prot_buf += sectors * bi->tuple_size; + intervals = bv->bv_len / bi->interval; + seed += intervals; + prot_buf += intervals * bi->tuple_size; kunmap_atomic(kaddr); } @@ -294,17 +294,17 @@ int bio_integrity_prep(struct bio *bio) unsigned long start, end; unsigned int len, nr_pages; unsigned int bytes, offset, i; - unsigned int sectors; + unsigned int intervals; bi = bdev_get_integrity(bio->bi_bdev); q = bdev_get_queue(bio->bi_bdev); BUG_ON(bi == NULL); BUG_ON(bio_integrity(bio)); - sectors = bio_integrity_hw_sectors(bi, bio_sectors(bio)); + intervals = bio_integrity_intervals(bi, bio_sectors(bio)); /* Allocate kernel buffer for protection data */ - len = sectors * bi->tuple_size; + len = intervals * bi->tuple_size; buf = kmalloc(len, GFP_NOIO | q->bounce_gfp); if (unlikely(buf == NULL)) { printk(KERN_ERR "could not allocate integrity buffer\n"); diff --git a/block/blk-integrity.c b/block/blk-integrity.c index 7ac17160ab69..3a83a7d08177 100644 --- a/block/blk-integrity.c +++ b/block/blk-integrity.c @@ -154,10 +154,10 @@ int blk_integrity_compare(struct gendisk *gd1, struct gendisk *gd2) if (!b1 || !b2) return -1; - if (b1->sector_size != b2->sector_size) { - printk(KERN_ERR "%s: %s/%s sector sz %u != %u\n", __func__, - gd1->disk_name, gd2->disk_name, - b1->sector_size, b2->sector_size); + if (b1->interval != b2->interval) { + pr_err("%s: %s/%s protection interval %u != %u\n", + __func__, gd1->disk_name, gd2->disk_name, + b1->interval, b2->interval); return -1; } @@ -407,7 +407,7 @@ int blk_integrity_register(struct gendisk *disk, struct blk_integrity *template) kobject_uevent(&bi->kobj, KOBJ_ADD); bi->flags |= INTEGRITY_FLAG_READ | INTEGRITY_FLAG_WRITE; - bi->sector_size = queue_logical_block_size(disk->queue); + bi->interval = queue_logical_block_size(disk->queue); disk->integrity = bi; } else bi = disk->integrity; diff --git a/drivers/scsi/sd_dif.c b/drivers/scsi/sd_dif.c index 38a7778631be..1600270a46e5 100644 --- a/drivers/scsi/sd_dif.c +++ b/drivers/scsi/sd_dif.c @@ -57,16 +57,16 @@ static void sd_dif_type1_generate(struct blk_integrity_exchg *bix, csum_fn *fn) { void *buf = bix->data_buf; struct sd_dif_tuple *sdt = bix->prot_buf; - sector_t sector = bix->sector; + sector_t seed = bix->seed; unsigned int i; - for (i = 0 ; i < bix->data_size ; i += bix->sector_size, sdt++) { - sdt->guard_tag = fn(buf, bix->sector_size); - sdt->ref_tag = cpu_to_be32(sector & 0xffffffff); + for (i = 0 ; i < bix->data_size ; i += bix->interval, sdt++) { + sdt->guard_tag = fn(buf, bix->interval); + sdt->ref_tag = cpu_to_be32(seed & 0xffffffff); sdt->app_tag = 0; - buf += bix->sector_size; - sector++; + buf += bix->interval; + seed++; } } @@ -84,35 +84,35 @@ static int sd_dif_type1_verify(struct blk_integrity_exchg *bix, csum_fn *fn) { void *buf = bix->data_buf; struct sd_dif_tuple *sdt = bix->prot_buf; - sector_t sector = bix->sector; + sector_t seed = bix->seed; unsigned int i; __u16 csum; - for (i = 0 ; i < bix->data_size ; i += bix->sector_size, sdt++) { + for (i = 0 ; i < bix->data_size ; i += bix->interval, sdt++) { /* Unwritten sectors */ if (sdt->app_tag == 0xffff) return 0; - if (be32_to_cpu(sdt->ref_tag) != (sector & 0xffffffff)) { + if (be32_to_cpu(sdt->ref_tag) != (seed & 0xffffffff)) { printk(KERN_ERR "%s: ref tag error on sector %lu (rcvd %u)\n", - bix->disk_name, (unsigned long)sector, + bix->disk_name, (unsigned long)seed, be32_to_cpu(sdt->ref_tag)); return -EIO; } - csum = fn(buf, bix->sector_size); + csum = fn(buf, bix->interval); if (sdt->guard_tag != csum) { printk(KERN_ERR "%s: guard tag error on sector %lu " \ "(rcvd %04x, data %04x)\n", bix->disk_name, - (unsigned long)sector, + (unsigned long)seed, be16_to_cpu(sdt->guard_tag), be16_to_cpu(csum)); return -EIO; } - buf += bix->sector_size; - sector++; + buf += bix->interval; + seed++; } return 0; @@ -155,12 +155,12 @@ static void sd_dif_type3_generate(struct blk_integrity_exchg *bix, csum_fn *fn) struct sd_dif_tuple *sdt = bix->prot_buf; unsigned int i; - for (i = 0 ; i < bix->data_size ; i += bix->sector_size, sdt++) { - sdt->guard_tag = fn(buf, bix->sector_size); + for (i = 0 ; i < bix->data_size ; i += bix->interval, sdt++) { + sdt->guard_tag = fn(buf, bix->interval); sdt->ref_tag = 0; sdt->app_tag = 0; - buf += bix->sector_size; + buf += bix->interval; } } @@ -178,27 +178,27 @@ static int sd_dif_type3_verify(struct blk_integrity_exchg *bix, csum_fn *fn) { void *buf = bix->data_buf; struct sd_dif_tuple *sdt = bix->prot_buf; - sector_t sector = bix->sector; + sector_t seed = bix->seed; unsigned int i; __u16 csum; - for (i = 0 ; i < bix->data_size ; i += bix->sector_size, sdt++) { + for (i = 0 ; i < bix->data_size ; i += bix->interval, sdt++) { /* Unwritten sectors */ if (sdt->app_tag == 0xffff && sdt->ref_tag == 0xffffffff) return 0; - csum = fn(buf, bix->sector_size); + csum = fn(buf, bix->interval); if (sdt->guard_tag != csum) { printk(KERN_ERR "%s: guard tag error on sector %lu " \ "(rcvd %04x, data %04x)\n", bix->disk_name, - (unsigned long)sector, + (unsigned long)seed, be16_to_cpu(sdt->guard_tag), be16_to_cpu(csum)); return -EIO; } - buf += bix->sector_size; - sector++; + buf += bix->interval; + seed++; } return 0; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 0bf5d79d9ba0..d364c42dbf17 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1464,9 +1464,9 @@ static inline uint64_t rq_io_start_time_ns(struct request *req) struct blk_integrity_exchg { void *prot_buf; void *data_buf; - sector_t sector; + sector_t seed; unsigned int data_size; - unsigned short sector_size; + unsigned short interval; const char *disk_name; }; @@ -1479,7 +1479,7 @@ struct blk_integrity { unsigned short flags; unsigned short tuple_size; - unsigned short sector_size; + unsigned short interval; unsigned short tag_size; const char *name; -- cgit v1.2.3 From 1859308853b19c4daf4afaab910d3d52ac1ec2ff Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:20:01 -0400 Subject: block: Clean up the code used to generate and verify integrity metadata Instead of the "operate" parameter we pass in a seed value and a pointer to a function that can be used to process the integrity metadata. The generation function is changed to have a return value to fit into this scheme. Signed-off-by: Martin K. Petersen Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/bio-integrity.c | 82 ++++++++++---------------------------- drivers/scsi/sd_dif.c | 106 ++++++++++++++++++++++++++----------------------- include/linux/bio.h | 12 ++++++ include/linux/blkdev.h | 9 ++--- 4 files changed, 94 insertions(+), 115 deletions(-) (limited to 'drivers') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index cf40837e7710..fe4de033b34c 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -207,69 +207,43 @@ static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi, } /** - * bio_integrity_generate_verify - Generate/verify integrity metadata for a bio + * bio_integrity_process - Process integrity metadata for a bio * @bio: bio to generate/verify integrity metadata for - * @operate: operate number, 1 for generate, 0 for verify + * @proc_fn: Pointer to the relevant processing function */ -static int bio_integrity_generate_verify(struct bio *bio, int operate) +static int bio_integrity_process(struct bio *bio, + integrity_processing_fn *proc_fn) { struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); - struct blk_integrity_exchg bix; + struct blk_integrity_iter iter; struct bio_vec *bv; struct bio_integrity_payload *bip = bio_integrity(bio); - sector_t seed; - unsigned int intervals, ret = 0, i; + unsigned int i, ret = 0; void *prot_buf = page_address(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset; - if (operate) - seed = bio->bi_iter.bi_sector; - else - seed = bip->bip_iter.bi_sector; - - bix.disk_name = bio->bi_bdev->bd_disk->disk_name; - bix.interval = bi->interval; + iter.disk_name = bio->bi_bdev->bd_disk->disk_name; + iter.interval = bi->interval; + iter.seed = bip_get_seed(bip); + iter.prot_buf = prot_buf; bio_for_each_segment_all(bv, bio, i) { void *kaddr = kmap_atomic(bv->bv_page); - bix.data_buf = kaddr + bv->bv_offset; - bix.data_size = bv->bv_len; - bix.prot_buf = prot_buf; - bix.seed = seed; - - if (operate) - bi->generate_fn(&bix); - else { - ret = bi->verify_fn(&bix); - if (ret) { - kunmap_atomic(kaddr); - return ret; - } - } - intervals = bv->bv_len / bi->interval; - seed += intervals; - prot_buf += intervals * bi->tuple_size; + iter.data_buf = kaddr + bv->bv_offset; + iter.data_size = bv->bv_len; + + ret = proc_fn(&iter); + if (ret) { + kunmap_atomic(kaddr); + return ret; + } kunmap_atomic(kaddr); } return ret; } -/** - * bio_integrity_generate - Generate integrity metadata for a bio - * @bio: bio to generate integrity metadata for - * - * Description: Generates integrity metadata for a bio by calling the - * block device's generation callback function. The bio must have a - * bip attached with enough room to accommodate the generated - * integrity metadata. - */ -static void bio_integrity_generate(struct bio *bio) -{ - bio_integrity_generate_verify(bio, 1); -} - /** * bio_integrity_prep - Prepare bio for integrity I/O * @bio: bio to prepare @@ -321,7 +295,7 @@ int bio_integrity_prep(struct bio *bio) bip->bip_owns_buf = 1; bip->bip_iter.bi_size = len; - bip->bip_iter.bi_sector = bio->bi_iter.bi_sector; + bip_set_seed(bip, bio->bi_iter.bi_sector); /* Map it */ offset = offset_in_page(buf); @@ -357,25 +331,12 @@ int bio_integrity_prep(struct bio *bio) /* Auto-generate integrity metadata if this is a write */ if (bio_data_dir(bio) == WRITE) - bio_integrity_generate(bio); + bio_integrity_process(bio, bi->generate_fn); return 0; } EXPORT_SYMBOL(bio_integrity_prep); -/** - * bio_integrity_verify - Verify integrity metadata for a bio - * @bio: bio to verify - * - * Description: This function is called to verify the integrity of a - * bio. The data in the bio io_vec is compared to the integrity - * metadata returned by the HBA. - */ -static int bio_integrity_verify(struct bio *bio) -{ - return bio_integrity_generate_verify(bio, 0); -} - /** * bio_integrity_verify_fn - Integrity I/O completion worker * @work: Work struct stored in bio to be verified @@ -389,9 +350,10 @@ static void bio_integrity_verify_fn(struct work_struct *work) struct bio_integrity_payload *bip = container_of(work, struct bio_integrity_payload, bip_work); struct bio *bio = bip->bip_bio; + struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); int error; - error = bio_integrity_verify(bio); + error = bio_integrity_process(bio, bi->verify_fn); /* Restore original bio completion handler */ bio->bi_end_io = bip->bip_end_io; diff --git a/drivers/scsi/sd_dif.c b/drivers/scsi/sd_dif.c index 1600270a46e5..801c41851a01 100644 --- a/drivers/scsi/sd_dif.c +++ b/drivers/scsi/sd_dif.c @@ -53,42 +53,44 @@ static __u16 sd_dif_ip_fn(void *data, unsigned int len) * Type 1 and Type 2 protection use the same format: 16 bit guard tag, * 16 bit app tag, 32 bit reference tag. */ -static void sd_dif_type1_generate(struct blk_integrity_exchg *bix, csum_fn *fn) +static void sd_dif_type1_generate(struct blk_integrity_iter *iter, csum_fn *fn) { - void *buf = bix->data_buf; - struct sd_dif_tuple *sdt = bix->prot_buf; - sector_t seed = bix->seed; + void *buf = iter->data_buf; + struct sd_dif_tuple *sdt = iter->prot_buf; + sector_t seed = iter->seed; unsigned int i; - for (i = 0 ; i < bix->data_size ; i += bix->interval, sdt++) { - sdt->guard_tag = fn(buf, bix->interval); + for (i = 0 ; i < iter->data_size ; i += iter->interval, sdt++) { + sdt->guard_tag = fn(buf, iter->interval); sdt->ref_tag = cpu_to_be32(seed & 0xffffffff); sdt->app_tag = 0; - buf += bix->interval; + buf += iter->interval; seed++; } } -static void sd_dif_type1_generate_crc(struct blk_integrity_exchg *bix) +static int sd_dif_type1_generate_crc(struct blk_integrity_iter *iter) { - sd_dif_type1_generate(bix, sd_dif_crc_fn); + sd_dif_type1_generate(iter, sd_dif_crc_fn); + return 0; } -static void sd_dif_type1_generate_ip(struct blk_integrity_exchg *bix) +static int sd_dif_type1_generate_ip(struct blk_integrity_iter *iter) { - sd_dif_type1_generate(bix, sd_dif_ip_fn); + sd_dif_type1_generate(iter, sd_dif_ip_fn); + return 0; } -static int sd_dif_type1_verify(struct blk_integrity_exchg *bix, csum_fn *fn) +static int sd_dif_type1_verify(struct blk_integrity_iter *iter, csum_fn *fn) { - void *buf = bix->data_buf; - struct sd_dif_tuple *sdt = bix->prot_buf; - sector_t seed = bix->seed; + void *buf = iter->data_buf; + struct sd_dif_tuple *sdt = iter->prot_buf; + sector_t seed = iter->seed; unsigned int i; __u16 csum; - for (i = 0 ; i < bix->data_size ; i += bix->interval, sdt++) { + for (i = 0 ; i < iter->data_size ; i += iter->interval, sdt++) { /* Unwritten sectors */ if (sdt->app_tag == 0xffff) return 0; @@ -96,36 +98,36 @@ static int sd_dif_type1_verify(struct blk_integrity_exchg *bix, csum_fn *fn) if (be32_to_cpu(sdt->ref_tag) != (seed & 0xffffffff)) { printk(KERN_ERR "%s: ref tag error on sector %lu (rcvd %u)\n", - bix->disk_name, (unsigned long)seed, + iter->disk_name, (unsigned long)seed, be32_to_cpu(sdt->ref_tag)); return -EIO; } - csum = fn(buf, bix->interval); + csum = fn(buf, iter->interval); if (sdt->guard_tag != csum) { printk(KERN_ERR "%s: guard tag error on sector %lu " \ - "(rcvd %04x, data %04x)\n", bix->disk_name, + "(rcvd %04x, data %04x)\n", iter->disk_name, (unsigned long)seed, be16_to_cpu(sdt->guard_tag), be16_to_cpu(csum)); return -EIO; } - buf += bix->interval; + buf += iter->interval; seed++; } return 0; } -static int sd_dif_type1_verify_crc(struct blk_integrity_exchg *bix) +static int sd_dif_type1_verify_crc(struct blk_integrity_iter *iter) { - return sd_dif_type1_verify(bix, sd_dif_crc_fn); + return sd_dif_type1_verify(iter, sd_dif_crc_fn); } -static int sd_dif_type1_verify_ip(struct blk_integrity_exchg *bix) +static int sd_dif_type1_verify_ip(struct blk_integrity_iter *iter) { - return sd_dif_type1_verify(bix, sd_dif_ip_fn); + return sd_dif_type1_verify(iter, sd_dif_ip_fn); } static struct blk_integrity dif_type1_integrity_crc = { @@ -149,69 +151,71 @@ static struct blk_integrity dif_type1_integrity_ip = { * Type 3 protection has a 16-bit guard tag and 16 + 32 bits of opaque * tag space. */ -static void sd_dif_type3_generate(struct blk_integrity_exchg *bix, csum_fn *fn) +static void sd_dif_type3_generate(struct blk_integrity_iter *iter, csum_fn *fn) { - void *buf = bix->data_buf; - struct sd_dif_tuple *sdt = bix->prot_buf; + void *buf = iter->data_buf; + struct sd_dif_tuple *sdt = iter->prot_buf; unsigned int i; - for (i = 0 ; i < bix->data_size ; i += bix->interval, sdt++) { - sdt->guard_tag = fn(buf, bix->interval); + for (i = 0 ; i < iter->data_size ; i += iter->interval, sdt++) { + sdt->guard_tag = fn(buf, iter->interval); sdt->ref_tag = 0; sdt->app_tag = 0; - buf += bix->interval; + buf += iter->interval; } } -static void sd_dif_type3_generate_crc(struct blk_integrity_exchg *bix) +static int sd_dif_type3_generate_crc(struct blk_integrity_iter *iter) { - sd_dif_type3_generate(bix, sd_dif_crc_fn); + sd_dif_type3_generate(iter, sd_dif_crc_fn); + return 0; } -static void sd_dif_type3_generate_ip(struct blk_integrity_exchg *bix) +static int sd_dif_type3_generate_ip(struct blk_integrity_iter *iter) { - sd_dif_type3_generate(bix, sd_dif_ip_fn); + sd_dif_type3_generate(iter, sd_dif_ip_fn); + return 0; } -static int sd_dif_type3_verify(struct blk_integrity_exchg *bix, csum_fn *fn) +static int sd_dif_type3_verify(struct blk_integrity_iter *iter, csum_fn *fn) { - void *buf = bix->data_buf; - struct sd_dif_tuple *sdt = bix->prot_buf; - sector_t seed = bix->seed; + void *buf = iter->data_buf; + struct sd_dif_tuple *sdt = iter->prot_buf; + sector_t seed = iter->seed; unsigned int i; __u16 csum; - for (i = 0 ; i < bix->data_size ; i += bix->interval, sdt++) { + for (i = 0 ; i < iter->data_size ; i += iter->interval, sdt++) { /* Unwritten sectors */ if (sdt->app_tag == 0xffff && sdt->ref_tag == 0xffffffff) return 0; - csum = fn(buf, bix->interval); + csum = fn(buf, iter->interval); if (sdt->guard_tag != csum) { printk(KERN_ERR "%s: guard tag error on sector %lu " \ - "(rcvd %04x, data %04x)\n", bix->disk_name, + "(rcvd %04x, data %04x)\n", iter->disk_name, (unsigned long)seed, be16_to_cpu(sdt->guard_tag), be16_to_cpu(csum)); return -EIO; } - buf += bix->interval; + buf += iter->interval; seed++; } return 0; } -static int sd_dif_type3_verify_crc(struct blk_integrity_exchg *bix) +static int sd_dif_type3_verify_crc(struct blk_integrity_iter *iter) { - return sd_dif_type3_verify(bix, sd_dif_crc_fn); + return sd_dif_type3_verify(iter, sd_dif_crc_fn); } -static int sd_dif_type3_verify_ip(struct blk_integrity_exchg *bix) +static int sd_dif_type3_verify_ip(struct blk_integrity_iter *iter) { - return sd_dif_type3_verify(bix, sd_dif_ip_fn); + return sd_dif_type3_verify(iter, sd_dif_ip_fn); } static struct blk_integrity dif_type3_integrity_crc = { @@ -310,6 +314,7 @@ void sd_dif_prepare(struct request *rq, sector_t hw_sector, phys = hw_sector & 0xffffffff; __rq_for_each_bio(bio, rq) { + struct bio_integrity_payload *bip = bio_integrity(bio); struct bio_vec iv; struct bvec_iter iter; unsigned int j; @@ -318,9 +323,9 @@ void sd_dif_prepare(struct request *rq, sector_t hw_sector, if (bio_flagged(bio, BIO_MAPPED_INTEGRITY)) break; - virt = bio_integrity(bio)->bip_iter.bi_sector & 0xffffffff; + virt = bip_get_seed(bip) & 0xffffffff; - bip_for_each_vec(iv, bio_integrity(bio), iter) { + bip_for_each_vec(iv, bip, iter) { sdt = kmap_atomic(iv.bv_page) + iv.bv_offset; @@ -366,12 +371,13 @@ void sd_dif_complete(struct scsi_cmnd *scmd, unsigned int good_bytes) phys >>= 3; __rq_for_each_bio(bio, scmd->request) { + struct bio_integrity_payload *bip = bio_integrity(bio); struct bio_vec iv; struct bvec_iter iter; - virt = bio_integrity(bio)->bip_iter.bi_sector & 0xffffffff; + virt = bip_get_seed(bip) & 0xffffffff; - bip_for_each_vec(iv, bio_integrity(bio), iter) { + bip_for_each_vec(iv, bip, iter) { sdt = kmap_atomic(iv.bv_page) + iv.bv_offset; diff --git a/include/linux/bio.h b/include/linux/bio.h index 448d8c052cb7..3fd36660fd10 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -322,6 +322,18 @@ struct bio_integrity_payload { struct bio_vec *bip_vec; struct bio_vec bip_inline_vecs[0];/* embedded bvec array */ }; + +static inline sector_t bip_get_seed(struct bio_integrity_payload *bip) +{ + return bip->bip_iter.bi_sector; +} + +static inline void bip_set_seed(struct bio_integrity_payload *bip, + sector_t seed) +{ + bip->bip_iter.bi_sector = seed; +} + #endif /* CONFIG_BLK_DEV_INTEGRITY */ extern void bio_trim(struct bio *bio, int offset, int size); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index d364c42dbf17..24c1e055b8a7 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1461,7 +1461,7 @@ static inline uint64_t rq_io_start_time_ns(struct request *req) #define INTEGRITY_FLAG_READ 2 /* verify data integrity on read */ #define INTEGRITY_FLAG_WRITE 4 /* generate data integrity on write */ -struct blk_integrity_exchg { +struct blk_integrity_iter { void *prot_buf; void *data_buf; sector_t seed; @@ -1470,12 +1470,11 @@ struct blk_integrity_exchg { const char *disk_name; }; -typedef void (integrity_gen_fn) (struct blk_integrity_exchg *); -typedef int (integrity_vrfy_fn) (struct blk_integrity_exchg *); +typedef int (integrity_processing_fn) (struct blk_integrity_iter *); struct blk_integrity { - integrity_gen_fn *generate_fn; - integrity_vrfy_fn *verify_fn; + integrity_processing_fn *generate_fn; + integrity_processing_fn *verify_fn; unsigned short flags; unsigned short tuple_size; -- cgit v1.2.3 From 3aec2f41a8baeb70aa77556a4e4dcec7d9d70b4d Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:20:03 -0400 Subject: block: Add a disk flag to block integrity profile So far we have relied on the app tag size to determine whether a disk has been formatted with T10 protection information or not. However, not all target devices provide application tag storage. Add a flag to the block integrity profile that indicates whether the disk has been formatted with protection information. Signed-off-by: Martin K. Petersen Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- Documentation/ABI/testing/sysfs-block | 8 ++++++++ block/blk-integrity.c | 12 ++++++++++++ drivers/scsi/sd_dif.c | 8 +++++++- include/linux/blkdev.h | 1 + 4 files changed, 28 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/Documentation/ABI/testing/sysfs-block b/Documentation/ABI/testing/sysfs-block index 279da08f7541..8df003963d99 100644 --- a/Documentation/ABI/testing/sysfs-block +++ b/Documentation/ABI/testing/sysfs-block @@ -53,6 +53,14 @@ Description: 512 bytes of data. +What: /sys/block//integrity/device_is_integrity_capable +Date: July 2014 +Contact: Martin K. Petersen +Description: + Indicates whether a storage device is capable of storing + integrity metadata. Set if the device is T10 PI-capable. + + What: /sys/block//integrity/write_generate Date: June 2008 Contact: Martin K. Petersen diff --git a/block/blk-integrity.c b/block/blk-integrity.c index a7436ccc936b..1c6ba442cd91 100644 --- a/block/blk-integrity.c +++ b/block/blk-integrity.c @@ -307,6 +307,12 @@ static ssize_t integrity_generate_show(struct blk_integrity *bi, char *page) return sprintf(page, "%d\n", (bi->flags & BLK_INTEGRITY_GENERATE) != 0); } +static ssize_t integrity_device_show(struct blk_integrity *bi, char *page) +{ + return sprintf(page, "%u\n", + (bi->flags & BLK_INTEGRITY_DEVICE_CAPABLE) != 0); +} + static struct integrity_sysfs_entry integrity_format_entry = { .attr = { .name = "format", .mode = S_IRUGO }, .show = integrity_format_show, @@ -329,11 +335,17 @@ static struct integrity_sysfs_entry integrity_generate_entry = { .store = integrity_generate_store, }; +static struct integrity_sysfs_entry integrity_device_entry = { + .attr = { .name = "device_is_integrity_capable", .mode = S_IRUGO }, + .show = integrity_device_show, +}; + static struct attribute *integrity_attrs[] = { &integrity_format_entry.attr, &integrity_tag_size_entry.attr, &integrity_verify_entry.attr, &integrity_generate_entry.attr, + &integrity_device_entry.attr, NULL, }; diff --git a/drivers/scsi/sd_dif.c b/drivers/scsi/sd_dif.c index 801c41851a01..1e971c6f8c2b 100644 --- a/drivers/scsi/sd_dif.c +++ b/drivers/scsi/sd_dif.c @@ -270,7 +270,13 @@ void sd_dif_config_host(struct scsi_disk *sdkp) "Enabling DIX %s protection\n", disk->integrity->name); /* Signal to block layer that we support sector tagging */ - if (dif && type && sdkp->ATO) { + if (dif && type) { + + disk->integrity->flags |= BLK_INTEGRITY_DEVICE_CAPABLE; + + if (!sdkp) + return; + if (type == SD_DIF_TYPE3_PROTECTION) disk->integrity->tag_size = sizeof(u16) + sizeof(u32); else diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index cf92eb031ae9..4600fc63e3fc 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1461,6 +1461,7 @@ static inline uint64_t rq_io_start_time_ns(struct request *req) enum blk_integrity_flags { BLK_INTEGRITY_VERIFY = 1 << 0, BLK_INTEGRITY_GENERATE = 1 << 1, + BLK_INTEGRITY_DEVICE_CAPABLE = 1 << 2, }; struct blk_integrity_iter { -- cgit v1.2.3 From b1f01388574c9329922f760fc2a7335c2d14b08b Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:20:04 -0400 Subject: block: Relocate bio integrity flags Move flags affecting the integrity code out of the bio bi_flags and into the block integrity payload. Signed-off-by: Martin K. Petersen Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/bio-integrity.c | 4 ++-- drivers/scsi/sd_dif.c | 4 ++-- include/linux/bio.h | 9 ++++++++- include/linux/blk_types.h | 6 ++---- 4 files changed, 14 insertions(+), 9 deletions(-) (limited to 'drivers') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index e64733bb29b6..26aa901b961f 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -100,7 +100,7 @@ void bio_integrity_free(struct bio *bio) struct bio_integrity_payload *bip = bio_integrity(bio); struct bio_set *bs = bio->bi_pool; - if (bip->bip_owns_buf) + if (bip->bip_flags & BIP_BLOCK_INTEGRITY) kfree(page_address(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset); @@ -293,7 +293,7 @@ int bio_integrity_prep(struct bio *bio) return -EIO; } - bip->bip_owns_buf = 1; + bip->bip_flags |= BIP_BLOCK_INTEGRITY; bip->bip_iter.bi_size = len; bip_set_seed(bip, bio->bi_iter.bi_sector); diff --git a/drivers/scsi/sd_dif.c b/drivers/scsi/sd_dif.c index 1e971c6f8c2b..4ce636fdc15f 100644 --- a/drivers/scsi/sd_dif.c +++ b/drivers/scsi/sd_dif.c @@ -326,7 +326,7 @@ void sd_dif_prepare(struct request *rq, sector_t hw_sector, unsigned int j; /* Already remapped? */ - if (bio_flagged(bio, BIO_MAPPED_INTEGRITY)) + if (bip->bip_flags & BIP_MAPPED_INTEGRITY) break; virt = bip_get_seed(bip) & 0xffffffff; @@ -347,7 +347,7 @@ void sd_dif_prepare(struct request *rq, sector_t hw_sector, kunmap_atomic(sdt); } - bio->bi_flags |= (1 << BIO_MAPPED_INTEGRITY); + bip->bip_flags |= BIP_MAPPED_INTEGRITY; } } diff --git a/include/linux/bio.h b/include/linux/bio.h index 3fd36660fd10..b508cf69206d 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -315,7 +315,7 @@ struct bio_integrity_payload { unsigned short bip_slab; /* slab the bip came from */ unsigned short bip_vcnt; /* # of integrity bio_vecs */ unsigned short bip_max_vcnt; /* integrity bio_vec slots */ - unsigned bip_owns_buf:1; /* should free bip_buf */ + unsigned short bip_flags; /* control flags */ struct work_struct bip_work; /* I/O completion */ @@ -323,6 +323,13 @@ struct bio_integrity_payload { struct bio_vec bip_inline_vecs[0];/* embedded bvec array */ }; +enum bip_flags { + BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */ + BIP_MAPPED_INTEGRITY = 1 << 1, /* ref tag has been remapped */ + BIP_CTRL_NOCHECK = 1 << 2, /* disable HBA integrity checking */ + BIP_DISK_NOCHECK = 1 << 3, /* disable disk integrity checking */ +}; + static inline sector_t bip_get_seed(struct bio_integrity_payload *bip) { return bip->bip_iter.bi_sector; diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 6a5d2f2de1b9..38bc008e4503 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -120,10 +120,8 @@ struct bio { #define BIO_USER_MAPPED 6 /* contains user pages */ #define BIO_EOPNOTSUPP 7 /* not supported */ #define BIO_NULL_MAPPED 8 /* contains invalid user pages */ -#define BIO_FS_INTEGRITY 9 /* fs owns integrity data, not block layer */ -#define BIO_QUIET 10 /* Make BIO Quiet */ -#define BIO_MAPPED_INTEGRITY 11/* integrity metadata has been remapped */ -#define BIO_SNAP_STABLE 12 /* bio data must be snapshotted during write */ +#define BIO_QUIET 9 /* Make BIO Quiet */ +#define BIO_SNAP_STABLE 10 /* bio data must be snapshotted during write */ /* * Flags starting here get preserved by bio_reset() - this includes -- cgit v1.2.3 From aae7df50190a640e51bfe11c93f94741ac82ff0b Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:20:05 -0400 Subject: block: Integrity checksum flag Make the choice of checksum a per-I/O property by introducing a flag that can be inspected by the SCSI layer. There are several reasons for this: 1. It allows us to switch choice of checksum without unloading and reloading the HBA driver. 2. During error recovery we need to be able to tell the HBA that checksums read from disk should not be verified and converted to IP checksums. 3. For error injection purposes we need to be able to write a bad guard tag to storage. Since the storage device only supports T10 CRC we need to be able to disable IP checksum conversion on the HBA. Signed-off-by: Martin K. Petersen Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/bio-integrity.c | 3 +++ drivers/scsi/sd_dif.c | 6 ++++-- include/linux/bio.h | 1 + include/linux/blkdev.h | 1 + 4 files changed, 9 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 26aa901b961f..8e0548484dd3 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -297,6 +297,9 @@ int bio_integrity_prep(struct bio *bio) bip->bip_iter.bi_size = len; bip_set_seed(bip, bio->bi_iter.bi_sector); + if (bi->flags & BLK_INTEGRITY_IP_CHECKSUM) + bip->bip_flags |= BIP_IP_CHECKSUM; + /* Map it */ offset = offset_in_page(buf); for (i = 0 ; i < nr_pages ; i++) { diff --git a/drivers/scsi/sd_dif.c b/drivers/scsi/sd_dif.c index 4ce636fdc15f..2198abee619e 100644 --- a/drivers/scsi/sd_dif.c +++ b/drivers/scsi/sd_dif.c @@ -255,12 +255,14 @@ void sd_dif_config_host(struct scsi_disk *sdkp) return; /* Enable DMA of protection information */ - if (scsi_host_get_guard(sdkp->device->host) & SHOST_DIX_GUARD_IP) + if (scsi_host_get_guard(sdkp->device->host) & SHOST_DIX_GUARD_IP) { if (type == SD_DIF_TYPE3_PROTECTION) blk_integrity_register(disk, &dif_type3_integrity_ip); else blk_integrity_register(disk, &dif_type1_integrity_ip); - else + + disk->integrity->flags |= BLK_INTEGRITY_IP_CHECKSUM; + } else if (type == SD_DIF_TYPE3_PROTECTION) blk_integrity_register(disk, &dif_type3_integrity_crc); else diff --git a/include/linux/bio.h b/include/linux/bio.h index b508cf69206d..14bff3fe56d4 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -328,6 +328,7 @@ enum bip_flags { BIP_MAPPED_INTEGRITY = 1 << 1, /* ref tag has been remapped */ BIP_CTRL_NOCHECK = 1 << 2, /* disable HBA integrity checking */ BIP_DISK_NOCHECK = 1 << 3, /* disable disk integrity checking */ + BIP_IP_CHECKSUM = 1 << 4, /* IP checksum */ }; static inline sector_t bip_get_seed(struct bio_integrity_payload *bip) diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 4600fc63e3fc..773df190a4ee 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1462,6 +1462,7 @@ enum blk_integrity_flags { BLK_INTEGRITY_VERIFY = 1 << 0, BLK_INTEGRITY_GENERATE = 1 << 1, BLK_INTEGRITY_DEVICE_CAPABLE = 1 << 2, + BLK_INTEGRITY_IP_CHECKSUM = 1 << 3, }; struct blk_integrity_iter { -- cgit v1.2.3 From 2341c2f8c33196d02cf5a721746eea4e3c06674a Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:20:07 -0400 Subject: block: Add T10 Protection Information functions The T10 Protection Information format is also used by some devices that do not go through the SCSI layer (virtual block devices, NVMe). Relocate the relevant functions to a block layer library that can be used without involving SCSI. Signed-off-by: Martin K. Petersen Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/Kconfig | 1 + block/Makefile | 4 +- block/t10-pi.c | 197 ++++++++++++++++++++++++++++++++++++ drivers/scsi/Kconfig | 1 - drivers/scsi/sd_dif.c | 241 ++++----------------------------------------- include/linux/crc-t10dif.h | 5 +- include/linux/t10-pi.h | 22 +++++ 7 files changed, 245 insertions(+), 226 deletions(-) create mode 100644 block/t10-pi.c create mode 100644 include/linux/t10-pi.h (limited to 'drivers') diff --git a/block/Kconfig b/block/Kconfig index 2429515c05c2..161491d0a879 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -77,6 +77,7 @@ config BLK_DEV_BSGLIB config BLK_DEV_INTEGRITY bool "Block layer data integrity support" + select CRC_T10DIF if BLK_DEV_INTEGRITY ---help--- Some storage devices allow extra information to be stored/retrieved to help protect the data. The block layer diff --git a/block/Makefile b/block/Makefile index a2ce6ac935ec..00ecc97629db 100644 --- a/block/Makefile +++ b/block/Makefile @@ -20,6 +20,6 @@ obj-$(CONFIG_IOSCHED_DEADLINE) += deadline-iosched.o obj-$(CONFIG_IOSCHED_CFQ) += cfq-iosched.o obj-$(CONFIG_BLOCK_COMPAT) += compat_ioctl.o -obj-$(CONFIG_BLK_DEV_INTEGRITY) += blk-integrity.o obj-$(CONFIG_BLK_CMDLINE_PARSER) += cmdline-parser.o -obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o +obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o blk-integrity.o t10-pi.o + diff --git a/block/t10-pi.c b/block/t10-pi.c new file mode 100644 index 000000000000..24d6e9715318 --- /dev/null +++ b/block/t10-pi.c @@ -0,0 +1,197 @@ +/* + * t10_pi.c - Functions for generating and verifying T10 Protection + * Information. + * + * Copyright (C) 2007, 2008, 2014 Oracle Corporation + * Written by: Martin K. Petersen + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License version + * 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, + * USA. + * + */ + +#include +#include +#include +#include + +typedef __be16 (csum_fn) (void *, unsigned int); + +static const __be16 APP_ESCAPE = (__force __be16) 0xffff; +static const __be32 REF_ESCAPE = (__force __be32) 0xffffffff; + +static __be16 t10_pi_crc_fn(void *data, unsigned int len) +{ + return cpu_to_be16(crc_t10dif(data, len)); +} + +static __be16 t10_pi_ip_fn(void *data, unsigned int len) +{ + return (__force __be16)ip_compute_csum(data, len); +} + +/* + * Type 1 and Type 2 protection use the same format: 16 bit guard tag, + * 16 bit app tag, 32 bit reference tag. Type 3 does not define the ref + * tag. + */ +static int t10_pi_generate(struct blk_integrity_iter *iter, csum_fn *fn, + unsigned int type) +{ + unsigned int i; + + for (i = 0 ; i < iter->data_size ; i += iter->interval) { + struct t10_pi_tuple *pi = iter->prot_buf; + + pi->guard_tag = fn(iter->data_buf, iter->interval); + pi->app_tag = 0; + + if (type == 1) + pi->ref_tag = cpu_to_be32(lower_32_bits(iter->seed)); + else + pi->ref_tag = 0; + + iter->data_buf += iter->interval; + iter->prot_buf += sizeof(struct t10_pi_tuple); + iter->seed++; + } + + return 0; +} + +static int t10_pi_verify(struct blk_integrity_iter *iter, csum_fn *fn, + unsigned int type) +{ + unsigned int i; + + for (i = 0 ; i < iter->data_size ; i += iter->interval) { + struct t10_pi_tuple *pi = iter->prot_buf; + __be16 csum; + + switch (type) { + case 1: + case 2: + if (pi->app_tag == APP_ESCAPE) + goto next; + + if (be32_to_cpu(pi->ref_tag) != + lower_32_bits(iter->seed)) { + pr_err("%s: ref tag error at location %llu " \ + "(rcvd %u)\n", iter->disk_name, + (unsigned long long) + iter->seed, be32_to_cpu(pi->ref_tag)); + return -EILSEQ; + } + break; + case 3: + if (pi->app_tag == APP_ESCAPE && + pi->ref_tag == REF_ESCAPE) + goto next; + break; + } + + csum = fn(iter->data_buf, iter->interval); + + if (pi->guard_tag != csum) { + pr_err("%s: guard tag error at sector %llu " \ + "(rcvd %04x, want %04x)\n", iter->disk_name, + (unsigned long long)iter->seed, + be16_to_cpu(pi->guard_tag), be16_to_cpu(csum)); + return -EILSEQ; + } + +next: + iter->data_buf += iter->interval; + iter->prot_buf += sizeof(struct t10_pi_tuple); + iter->seed++; + } + + return 0; +} + +static int t10_pi_type1_generate_crc(struct blk_integrity_iter *iter) +{ + return t10_pi_generate(iter, t10_pi_crc_fn, 1); +} + +static int t10_pi_type1_generate_ip(struct blk_integrity_iter *iter) +{ + return t10_pi_generate(iter, t10_pi_ip_fn, 1); +} + +static int t10_pi_type1_verify_crc(struct blk_integrity_iter *iter) +{ + return t10_pi_verify(iter, t10_pi_crc_fn, 1); +} + +static int t10_pi_type1_verify_ip(struct blk_integrity_iter *iter) +{ + return t10_pi_verify(iter, t10_pi_ip_fn, 1); +} + +static int t10_pi_type3_generate_crc(struct blk_integrity_iter *iter) +{ + return t10_pi_generate(iter, t10_pi_crc_fn, 3); +} + +static int t10_pi_type3_generate_ip(struct blk_integrity_iter *iter) +{ + return t10_pi_generate(iter, t10_pi_ip_fn, 3); +} + +static int t10_pi_type3_verify_crc(struct blk_integrity_iter *iter) +{ + return t10_pi_verify(iter, t10_pi_crc_fn, 3); +} + +static int t10_pi_type3_verify_ip(struct blk_integrity_iter *iter) +{ + return t10_pi_verify(iter, t10_pi_ip_fn, 3); +} + +struct blk_integrity t10_pi_type1_crc = { + .name = "T10-DIF-TYPE1-CRC", + .generate_fn = t10_pi_type1_generate_crc, + .verify_fn = t10_pi_type1_verify_crc, + .tuple_size = sizeof(struct t10_pi_tuple), + .tag_size = 0, +}; +EXPORT_SYMBOL(t10_pi_type1_crc); + +struct blk_integrity t10_pi_type1_ip = { + .name = "T10-DIF-TYPE1-IP", + .generate_fn = t10_pi_type1_generate_ip, + .verify_fn = t10_pi_type1_verify_ip, + .tuple_size = sizeof(struct t10_pi_tuple), + .tag_size = 0, +}; +EXPORT_SYMBOL(t10_pi_type1_ip); + +struct blk_integrity t10_pi_type3_crc = { + .name = "T10-DIF-TYPE3-CRC", + .generate_fn = t10_pi_type3_generate_crc, + .verify_fn = t10_pi_type3_verify_crc, + .tuple_size = sizeof(struct t10_pi_tuple), + .tag_size = 0, +}; +EXPORT_SYMBOL(t10_pi_type3_crc); + +struct blk_integrity t10_pi_type3_ip = { + .name = "T10-DIF-TYPE3-IP", + .generate_fn = t10_pi_type3_generate_ip, + .verify_fn = t10_pi_type3_verify_ip, + .tuple_size = sizeof(struct t10_pi_tuple), + .tag_size = 0, +}; +EXPORT_SYMBOL(t10_pi_type3_ip); diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig index 18a3358eb1d4..9ece13f922a6 100644 --- a/drivers/scsi/Kconfig +++ b/drivers/scsi/Kconfig @@ -62,7 +62,6 @@ comment "SCSI support type (disk, tape, CD-ROM)" config BLK_DEV_SD tristate "SCSI disk support" depends on SCSI - select CRC_T10DIF if BLK_DEV_INTEGRITY ---help--- If you want to use SCSI hard disks, Fibre Channel disks, Serial ATA (SATA) or Parallel ATA (PATA) hard disks, diff --git a/drivers/scsi/sd_dif.c b/drivers/scsi/sd_dif.c index 2198abee619e..b7eaeadc18f9 100644 --- a/drivers/scsi/sd_dif.c +++ b/drivers/scsi/sd_dif.c @@ -21,7 +21,7 @@ */ #include -#include +#include #include #include @@ -33,207 +33,8 @@ #include #include -#include - #include "sd.h" -typedef __u16 (csum_fn) (void *, unsigned int); - -static __u16 sd_dif_crc_fn(void *data, unsigned int len) -{ - return cpu_to_be16(crc_t10dif(data, len)); -} - -static __u16 sd_dif_ip_fn(void *data, unsigned int len) -{ - return ip_compute_csum(data, len); -} - -/* - * Type 1 and Type 2 protection use the same format: 16 bit guard tag, - * 16 bit app tag, 32 bit reference tag. - */ -static void sd_dif_type1_generate(struct blk_integrity_iter *iter, csum_fn *fn) -{ - void *buf = iter->data_buf; - struct sd_dif_tuple *sdt = iter->prot_buf; - sector_t seed = iter->seed; - unsigned int i; - - for (i = 0 ; i < iter->data_size ; i += iter->interval, sdt++) { - sdt->guard_tag = fn(buf, iter->interval); - sdt->ref_tag = cpu_to_be32(seed & 0xffffffff); - sdt->app_tag = 0; - - buf += iter->interval; - seed++; - } -} - -static int sd_dif_type1_generate_crc(struct blk_integrity_iter *iter) -{ - sd_dif_type1_generate(iter, sd_dif_crc_fn); - return 0; -} - -static int sd_dif_type1_generate_ip(struct blk_integrity_iter *iter) -{ - sd_dif_type1_generate(iter, sd_dif_ip_fn); - return 0; -} - -static int sd_dif_type1_verify(struct blk_integrity_iter *iter, csum_fn *fn) -{ - void *buf = iter->data_buf; - struct sd_dif_tuple *sdt = iter->prot_buf; - sector_t seed = iter->seed; - unsigned int i; - __u16 csum; - - for (i = 0 ; i < iter->data_size ; i += iter->interval, sdt++) { - /* Unwritten sectors */ - if (sdt->app_tag == 0xffff) - return 0; - - if (be32_to_cpu(sdt->ref_tag) != (seed & 0xffffffff)) { - printk(KERN_ERR - "%s: ref tag error on sector %lu (rcvd %u)\n", - iter->disk_name, (unsigned long)seed, - be32_to_cpu(sdt->ref_tag)); - return -EIO; - } - - csum = fn(buf, iter->interval); - - if (sdt->guard_tag != csum) { - printk(KERN_ERR "%s: guard tag error on sector %lu " \ - "(rcvd %04x, data %04x)\n", iter->disk_name, - (unsigned long)seed, - be16_to_cpu(sdt->guard_tag), be16_to_cpu(csum)); - return -EIO; - } - - buf += iter->interval; - seed++; - } - - return 0; -} - -static int sd_dif_type1_verify_crc(struct blk_integrity_iter *iter) -{ - return sd_dif_type1_verify(iter, sd_dif_crc_fn); -} - -static int sd_dif_type1_verify_ip(struct blk_integrity_iter *iter) -{ - return sd_dif_type1_verify(iter, sd_dif_ip_fn); -} - -static struct blk_integrity dif_type1_integrity_crc = { - .name = "T10-DIF-TYPE1-CRC", - .generate_fn = sd_dif_type1_generate_crc, - .verify_fn = sd_dif_type1_verify_crc, - .tuple_size = sizeof(struct sd_dif_tuple), - .tag_size = 0, -}; - -static struct blk_integrity dif_type1_integrity_ip = { - .name = "T10-DIF-TYPE1-IP", - .generate_fn = sd_dif_type1_generate_ip, - .verify_fn = sd_dif_type1_verify_ip, - .tuple_size = sizeof(struct sd_dif_tuple), - .tag_size = 0, -}; - - -/* - * Type 3 protection has a 16-bit guard tag and 16 + 32 bits of opaque - * tag space. - */ -static void sd_dif_type3_generate(struct blk_integrity_iter *iter, csum_fn *fn) -{ - void *buf = iter->data_buf; - struct sd_dif_tuple *sdt = iter->prot_buf; - unsigned int i; - - for (i = 0 ; i < iter->data_size ; i += iter->interval, sdt++) { - sdt->guard_tag = fn(buf, iter->interval); - sdt->ref_tag = 0; - sdt->app_tag = 0; - - buf += iter->interval; - } -} - -static int sd_dif_type3_generate_crc(struct blk_integrity_iter *iter) -{ - sd_dif_type3_generate(iter, sd_dif_crc_fn); - return 0; -} - -static int sd_dif_type3_generate_ip(struct blk_integrity_iter *iter) -{ - sd_dif_type3_generate(iter, sd_dif_ip_fn); - return 0; -} - -static int sd_dif_type3_verify(struct blk_integrity_iter *iter, csum_fn *fn) -{ - void *buf = iter->data_buf; - struct sd_dif_tuple *sdt = iter->prot_buf; - sector_t seed = iter->seed; - unsigned int i; - __u16 csum; - - for (i = 0 ; i < iter->data_size ; i += iter->interval, sdt++) { - /* Unwritten sectors */ - if (sdt->app_tag == 0xffff && sdt->ref_tag == 0xffffffff) - return 0; - - csum = fn(buf, iter->interval); - - if (sdt->guard_tag != csum) { - printk(KERN_ERR "%s: guard tag error on sector %lu " \ - "(rcvd %04x, data %04x)\n", iter->disk_name, - (unsigned long)seed, - be16_to_cpu(sdt->guard_tag), be16_to_cpu(csum)); - return -EIO; - } - - buf += iter->interval; - seed++; - } - - return 0; -} - -static int sd_dif_type3_verify_crc(struct blk_integrity_iter *iter) -{ - return sd_dif_type3_verify(iter, sd_dif_crc_fn); -} - -static int sd_dif_type3_verify_ip(struct blk_integrity_iter *iter) -{ - return sd_dif_type3_verify(iter, sd_dif_ip_fn); -} - -static struct blk_integrity dif_type3_integrity_crc = { - .name = "T10-DIF-TYPE3-CRC", - .generate_fn = sd_dif_type3_generate_crc, - .verify_fn = sd_dif_type3_verify_crc, - .tuple_size = sizeof(struct sd_dif_tuple), - .tag_size = 0, -}; - -static struct blk_integrity dif_type3_integrity_ip = { - .name = "T10-DIF-TYPE3-IP", - .generate_fn = sd_dif_type3_generate_ip, - .verify_fn = sd_dif_type3_verify_ip, - .tuple_size = sizeof(struct sd_dif_tuple), - .tag_size = 0, -}; - /* * Configure exchange of protection information between OS and HBA. */ @@ -257,16 +58,16 @@ void sd_dif_config_host(struct scsi_disk *sdkp) /* Enable DMA of protection information */ if (scsi_host_get_guard(sdkp->device->host) & SHOST_DIX_GUARD_IP) { if (type == SD_DIF_TYPE3_PROTECTION) - blk_integrity_register(disk, &dif_type3_integrity_ip); + blk_integrity_register(disk, &t10_pi_type3_ip); else - blk_integrity_register(disk, &dif_type1_integrity_ip); + blk_integrity_register(disk, &t10_pi_type1_ip); disk->integrity->flags |= BLK_INTEGRITY_IP_CHECKSUM; } else if (type == SD_DIF_TYPE3_PROTECTION) - blk_integrity_register(disk, &dif_type3_integrity_crc); + blk_integrity_register(disk, &t10_pi_type3_crc); else - blk_integrity_register(disk, &dif_type1_integrity_crc); + blk_integrity_register(disk, &t10_pi_type1_crc); sd_printk(KERN_NOTICE, sdkp, "Enabling DIX %s protection\n", disk->integrity->name); @@ -308,10 +109,10 @@ void sd_dif_config_host(struct scsi_disk *sdkp) void sd_dif_prepare(struct request *rq, sector_t hw_sector, unsigned int sector_sz) { - const int tuple_sz = sizeof(struct sd_dif_tuple); + const int tuple_sz = sizeof(struct t10_pi_tuple); struct bio *bio; struct scsi_disk *sdkp; - struct sd_dif_tuple *sdt; + struct t10_pi_tuple *pi; u32 phys, virt; sdkp = rq->bio->bi_bdev->bd_disk->private_data; @@ -334,19 +135,18 @@ void sd_dif_prepare(struct request *rq, sector_t hw_sector, virt = bip_get_seed(bip) & 0xffffffff; bip_for_each_vec(iv, bip, iter) { - sdt = kmap_atomic(iv.bv_page) - + iv.bv_offset; + pi = kmap_atomic(iv.bv_page) + iv.bv_offset; - for (j = 0; j < iv.bv_len; j += tuple_sz, sdt++) { + for (j = 0; j < iv.bv_len; j += tuple_sz, pi++) { - if (be32_to_cpu(sdt->ref_tag) == virt) - sdt->ref_tag = cpu_to_be32(phys); + if (be32_to_cpu(pi->ref_tag) == virt) + pi->ref_tag = cpu_to_be32(phys); virt++; phys++; } - kunmap_atomic(sdt); + kunmap_atomic(pi); } bip->bip_flags |= BIP_MAPPED_INTEGRITY; @@ -359,10 +159,10 @@ void sd_dif_prepare(struct request *rq, sector_t hw_sector, */ void sd_dif_complete(struct scsi_cmnd *scmd, unsigned int good_bytes) { - const int tuple_sz = sizeof(struct sd_dif_tuple); + const int tuple_sz = sizeof(struct t10_pi_tuple); struct scsi_disk *sdkp; struct bio *bio; - struct sd_dif_tuple *sdt; + struct t10_pi_tuple *pi; unsigned int j, sectors, sector_sz; u32 phys, virt; @@ -386,25 +186,24 @@ void sd_dif_complete(struct scsi_cmnd *scmd, unsigned int good_bytes) virt = bip_get_seed(bip) & 0xffffffff; bip_for_each_vec(iv, bip, iter) { - sdt = kmap_atomic(iv.bv_page) - + iv.bv_offset; + pi = kmap_atomic(iv.bv_page) + iv.bv_offset; - for (j = 0; j < iv.bv_len; j += tuple_sz, sdt++) { + for (j = 0; j < iv.bv_len; j += tuple_sz, pi++) { if (sectors == 0) { - kunmap_atomic(sdt); + kunmap_atomic(pi); return; } - if (be32_to_cpu(sdt->ref_tag) == phys) - sdt->ref_tag = cpu_to_be32(virt); + if (be32_to_cpu(pi->ref_tag) == phys) + pi->ref_tag = cpu_to_be32(virt); virt++; phys++; sectors--; } - kunmap_atomic(sdt); + kunmap_atomic(pi); } } } diff --git a/include/linux/crc-t10dif.h b/include/linux/crc-t10dif.h index b3cb71f0d3b0..cf53d0773ce3 100644 --- a/include/linux/crc-t10dif.h +++ b/include/linux/crc-t10dif.h @@ -6,7 +6,8 @@ #define CRC_T10DIF_DIGEST_SIZE 2 #define CRC_T10DIF_BLOCK_SIZE 1 -__u16 crc_t10dif_generic(__u16 crc, const unsigned char *buffer, size_t len); -__u16 crc_t10dif(unsigned char const *, size_t); +extern __u16 crc_t10dif_generic(__u16 crc, const unsigned char *buffer, + size_t len); +extern __u16 crc_t10dif(unsigned char const *, size_t); #endif diff --git a/include/linux/t10-pi.h b/include/linux/t10-pi.h new file mode 100644 index 000000000000..6a8b9942632d --- /dev/null +++ b/include/linux/t10-pi.h @@ -0,0 +1,22 @@ +#ifndef _LINUX_T10_PI_H +#define _LINUX_T10_PI_H + +#include +#include + +/* + * T10 Protection Information tuple. + */ +struct t10_pi_tuple { + __be16 guard_tag; /* Checksum */ + __be16 app_tag; /* Opaque storage */ + __be32 ref_tag; /* Target LBA or indirect LBA */ +}; + + +extern struct blk_integrity t10_pi_type1_crc; +extern struct blk_integrity t10_pi_type1_ip; +extern struct blk_integrity t10_pi_type3_crc; +extern struct blk_integrity t10_pi_type3_ip; + +#endif -- cgit v1.2.3 From c611529e7cd3465ec0eada0f44200e8420c38908 Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:20:08 -0400 Subject: sd: Honor block layer integrity handling flags A set of flags introduced in the block layer enable better control over how protection information is handled. These flags are useful for both error injection and data recovery purposes. Checking can be enabled and disabled for controller and disk, and the guard tag format is now a per-I/O property. Update sd_protect_op to communicate the relevant information to the low-level device driver via a set of flags in scsi_cmnd. Signed-off-by: Martin K. Petersen Reviewed-by: Sagi Grimberg Acked-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/scsi/sd.c | 73 ++++++++++++++++++++++++++++-------------------- drivers/scsi/sd.h | 66 +++++++++++++++++++++++++++++++++++++++++-- drivers/scsi/sd_dif.c | 23 ++++++--------- include/linux/bio.h | 33 ++++++++++++++++------ include/scsi/scsi_cmnd.h | 36 +++++++++++++++++------- 5 files changed, 166 insertions(+), 65 deletions(-) (limited to 'drivers') diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index 2c2041ca4b70..9f7099f4b537 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -610,29 +610,44 @@ static void scsi_disk_put(struct scsi_disk *sdkp) mutex_unlock(&sd_ref_mutex); } -static void sd_prot_op(struct scsi_cmnd *scmd, unsigned int dif) -{ - unsigned int prot_op = SCSI_PROT_NORMAL; - unsigned int dix = scsi_prot_sg_count(scmd); - - if (scmd->sc_data_direction == DMA_FROM_DEVICE) { - if (dif && dix) - prot_op = SCSI_PROT_READ_PASS; - else if (dif && !dix) - prot_op = SCSI_PROT_READ_STRIP; - else if (!dif && dix) - prot_op = SCSI_PROT_READ_INSERT; - } else { - if (dif && dix) - prot_op = SCSI_PROT_WRITE_PASS; - else if (dif && !dix) - prot_op = SCSI_PROT_WRITE_INSERT; - else if (!dif && dix) - prot_op = SCSI_PROT_WRITE_STRIP; + + +static unsigned char sd_setup_protect_cmnd(struct scsi_cmnd *scmd, + unsigned int dix, unsigned int dif) +{ + struct bio *bio = scmd->request->bio; + unsigned int prot_op = sd_prot_op(rq_data_dir(scmd->request), dix, dif); + unsigned int protect = 0; + + if (dix) { /* DIX Type 0, 1, 2, 3 */ + if (bio_integrity_flagged(bio, BIP_IP_CHECKSUM)) + scmd->prot_flags |= SCSI_PROT_IP_CHECKSUM; + + if (bio_integrity_flagged(bio, BIP_CTRL_NOCHECK) == false) + scmd->prot_flags |= SCSI_PROT_GUARD_CHECK; + } + + if (dif != SD_DIF_TYPE3_PROTECTION) { /* DIX/DIF Type 0, 1, 2 */ + scmd->prot_flags |= SCSI_PROT_REF_INCREMENT; + + if (bio_integrity_flagged(bio, BIP_CTRL_NOCHECK) == false) + scmd->prot_flags |= SCSI_PROT_REF_CHECK; + } + + if (dif) { /* DIX/DIF Type 1, 2, 3 */ + scmd->prot_flags |= SCSI_PROT_TRANSFER_PI; + + if (bio_integrity_flagged(bio, BIP_DISK_NOCHECK)) + protect = 3 << 5; /* Disable target PI checking */ + else + protect = 1 << 5; /* Enable target PI checking */ } scsi_set_prot_op(scmd, prot_op); scsi_set_prot_type(scmd, dif); + scmd->prot_flags &= sd_prot_flag_mask(prot_op); + + return protect; } static void sd_config_discard(struct scsi_disk *sdkp, unsigned int mode) @@ -893,7 +908,8 @@ static int sd_setup_read_write_cmnd(struct scsi_cmnd *SCpnt) sector_t block = blk_rq_pos(rq); sector_t threshold; unsigned int this_count = blk_rq_sectors(rq); - int ret, host_dif; + unsigned int dif, dix; + int ret; unsigned char protect; ret = scsi_init_io(SCpnt, GFP_ATOMIC); @@ -995,7 +1011,7 @@ static int sd_setup_read_write_cmnd(struct scsi_cmnd *SCpnt) SCpnt->cmnd[0] = WRITE_6; if (blk_integrity_rq(rq)) - sd_dif_prepare(rq, block, sdp->sector_size); + sd_dif_prepare(SCpnt); } else if (rq_data_dir(rq) == READ) { SCpnt->cmnd[0] = READ_6; @@ -1010,14 +1026,15 @@ static int sd_setup_read_write_cmnd(struct scsi_cmnd *SCpnt) "writing" : "reading", this_count, blk_rq_sectors(rq))); - /* Set RDPROTECT/WRPROTECT if disk is formatted with DIF */ - host_dif = scsi_host_dif_capable(sdp->host, sdkp->protection_type); - if (host_dif) - protect = 1 << 5; + dix = scsi_prot_sg_count(SCpnt); + dif = scsi_host_dif_capable(SCpnt->device->host, sdkp->protection_type); + + if (dif || dix) + protect = sd_setup_protect_cmnd(SCpnt, dix, dif); else protect = 0; - if (host_dif == SD_DIF_TYPE2_PROTECTION) { + if (protect && sdkp->protection_type == SD_DIF_TYPE2_PROTECTION) { SCpnt->cmnd = mempool_alloc(sd_cdb_pool, GFP_ATOMIC); if (unlikely(SCpnt->cmnd == NULL)) { @@ -1102,10 +1119,6 @@ static int sd_setup_read_write_cmnd(struct scsi_cmnd *SCpnt) } SCpnt->sdb.length = this_count * sdp->sector_size; - /* If DIF or DIX is enabled, tell HBA how to handle request */ - if (host_dif || scsi_prot_sg_count(SCpnt)) - sd_prot_op(SCpnt, host_dif); - /* * We shouldn't disconnect in the middle of a sector, so with a dumb * host adapter, it's safe to assume that we can at least transfer diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h index 4c3ab8377fd3..467377884b63 100644 --- a/drivers/scsi/sd.h +++ b/drivers/scsi/sd.h @@ -166,6 +166,68 @@ enum sd_dif_target_protection_types { SD_DIF_TYPE3_PROTECTION = 0x3, }; +/* + * Look up the DIX operation based on whether the command is read or + * write and whether dix and dif are enabled. + */ +static inline unsigned int sd_prot_op(bool write, bool dix, bool dif) +{ + /* Lookup table: bit 2 (write), bit 1 (dix), bit 0 (dif) */ + const unsigned int ops[] = { /* wrt dix dif */ + SCSI_PROT_NORMAL, /* 0 0 0 */ + SCSI_PROT_READ_STRIP, /* 0 0 1 */ + SCSI_PROT_READ_INSERT, /* 0 1 0 */ + SCSI_PROT_READ_PASS, /* 0 1 1 */ + SCSI_PROT_NORMAL, /* 1 0 0 */ + SCSI_PROT_WRITE_INSERT, /* 1 0 1 */ + SCSI_PROT_WRITE_STRIP, /* 1 1 0 */ + SCSI_PROT_WRITE_PASS, /* 1 1 1 */ + }; + + return ops[write << 2 | dix << 1 | dif]; +} + +/* + * Returns a mask of the protection flags that are valid for a given DIX + * operation. + */ +static inline unsigned int sd_prot_flag_mask(unsigned int prot_op) +{ + const unsigned int flag_mask[] = { + [SCSI_PROT_NORMAL] = 0, + + [SCSI_PROT_READ_STRIP] = SCSI_PROT_TRANSFER_PI | + SCSI_PROT_GUARD_CHECK | + SCSI_PROT_REF_CHECK | + SCSI_PROT_REF_INCREMENT, + + [SCSI_PROT_READ_INSERT] = SCSI_PROT_REF_INCREMENT | + SCSI_PROT_IP_CHECKSUM, + + [SCSI_PROT_READ_PASS] = SCSI_PROT_TRANSFER_PI | + SCSI_PROT_GUARD_CHECK | + SCSI_PROT_REF_CHECK | + SCSI_PROT_REF_INCREMENT | + SCSI_PROT_IP_CHECKSUM, + + [SCSI_PROT_WRITE_INSERT] = SCSI_PROT_TRANSFER_PI | + SCSI_PROT_REF_INCREMENT, + + [SCSI_PROT_WRITE_STRIP] = SCSI_PROT_GUARD_CHECK | + SCSI_PROT_REF_CHECK | + SCSI_PROT_REF_INCREMENT | + SCSI_PROT_IP_CHECKSUM, + + [SCSI_PROT_WRITE_PASS] = SCSI_PROT_TRANSFER_PI | + SCSI_PROT_GUARD_CHECK | + SCSI_PROT_REF_CHECK | + SCSI_PROT_REF_INCREMENT | + SCSI_PROT_IP_CHECKSUM, + }; + + return flag_mask[prot_op]; +} + /* * Data Integrity Field tuple. */ @@ -178,7 +240,7 @@ struct sd_dif_tuple { #ifdef CONFIG_BLK_DEV_INTEGRITY extern void sd_dif_config_host(struct scsi_disk *); -extern void sd_dif_prepare(struct request *rq, sector_t, unsigned int); +extern void sd_dif_prepare(struct scsi_cmnd *scmd); extern void sd_dif_complete(struct scsi_cmnd *, unsigned int); #else /* CONFIG_BLK_DEV_INTEGRITY */ @@ -186,7 +248,7 @@ extern void sd_dif_complete(struct scsi_cmnd *, unsigned int); static inline void sd_dif_config_host(struct scsi_disk *disk) { } -static inline int sd_dif_prepare(struct request *rq, sector_t s, unsigned int a) +static inline int sd_dif_prepare(struct scsi_cmnd *scmd) { return 0; } diff --git a/drivers/scsi/sd_dif.c b/drivers/scsi/sd_dif.c index b7eaeadc18f9..14c7d42a11c2 100644 --- a/drivers/scsi/sd_dif.c +++ b/drivers/scsi/sd_dif.c @@ -106,8 +106,7 @@ void sd_dif_config_host(struct scsi_disk *sdkp) * * Type 3 does not have a reference tag so no remapping is required. */ -void sd_dif_prepare(struct request *rq, sector_t hw_sector, - unsigned int sector_sz) +void sd_dif_prepare(struct scsi_cmnd *scmd) { const int tuple_sz = sizeof(struct t10_pi_tuple); struct bio *bio; @@ -115,14 +114,14 @@ void sd_dif_prepare(struct request *rq, sector_t hw_sector, struct t10_pi_tuple *pi; u32 phys, virt; - sdkp = rq->bio->bi_bdev->bd_disk->private_data; + sdkp = scsi_disk(scmd->request->rq_disk); if (sdkp->protection_type == SD_DIF_TYPE3_PROTECTION) return; - phys = hw_sector & 0xffffffff; + phys = scsi_prot_ref_tag(scmd); - __rq_for_each_bio(bio, rq) { + __rq_for_each_bio(bio, scmd->request) { struct bio_integrity_payload *bip = bio_integrity(bio); struct bio_vec iv; struct bvec_iter iter; @@ -163,7 +162,7 @@ void sd_dif_complete(struct scsi_cmnd *scmd, unsigned int good_bytes) struct scsi_disk *sdkp; struct bio *bio; struct t10_pi_tuple *pi; - unsigned int j, sectors, sector_sz; + unsigned int j, intervals; u32 phys, virt; sdkp = scsi_disk(scmd->request->rq_disk); @@ -171,12 +170,8 @@ void sd_dif_complete(struct scsi_cmnd *scmd, unsigned int good_bytes) if (sdkp->protection_type == SD_DIF_TYPE3_PROTECTION || good_bytes == 0) return; - sector_sz = scmd->device->sector_size; - sectors = good_bytes / sector_sz; - - phys = blk_rq_pos(scmd->request) & 0xffffffff; - if (sector_sz == 4096) - phys >>= 3; + intervals = good_bytes / scsi_prot_interval(scmd); + phys = scsi_prot_ref_tag(scmd); __rq_for_each_bio(bio, scmd->request) { struct bio_integrity_payload *bip = bio_integrity(bio); @@ -190,7 +185,7 @@ void sd_dif_complete(struct scsi_cmnd *scmd, unsigned int good_bytes) for (j = 0; j < iv.bv_len; j += tuple_sz, pi++) { - if (sectors == 0) { + if (intervals == 0) { kunmap_atomic(pi); return; } @@ -200,7 +195,7 @@ void sd_dif_complete(struct scsi_cmnd *scmd, unsigned int good_bytes) virt++; phys++; - sectors--; + intervals--; } kunmap_atomic(pi); diff --git a/include/linux/bio.h b/include/linux/bio.h index 14bff3fe56d4..ce6b75964b71 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -292,6 +292,14 @@ static inline unsigned bio_segments(struct bio *bio) */ #define bio_get(bio) atomic_inc(&(bio)->bi_cnt) +enum bip_flags { + BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */ + BIP_MAPPED_INTEGRITY = 1 << 1, /* ref tag has been remapped */ + BIP_CTRL_NOCHECK = 1 << 2, /* disable HBA integrity checking */ + BIP_DISK_NOCHECK = 1 << 3, /* disable disk integrity checking */ + BIP_IP_CHECKSUM = 1 << 4, /* IP checksum */ +}; + #if defined(CONFIG_BLK_DEV_INTEGRITY) static inline struct bio_integrity_payload *bio_integrity(struct bio *bio) @@ -323,13 +331,15 @@ struct bio_integrity_payload { struct bio_vec bip_inline_vecs[0];/* embedded bvec array */ }; -enum bip_flags { - BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */ - BIP_MAPPED_INTEGRITY = 1 << 1, /* ref tag has been remapped */ - BIP_CTRL_NOCHECK = 1 << 2, /* disable HBA integrity checking */ - BIP_DISK_NOCHECK = 1 << 3, /* disable disk integrity checking */ - BIP_IP_CHECKSUM = 1 << 4, /* IP checksum */ -}; +static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag) +{ + struct bio_integrity_payload *bip = bio_integrity(bio); + + if (bip) + return bip->bip_flags & flag; + + return false; +} static inline sector_t bip_get_seed(struct bio_integrity_payload *bip) { @@ -701,9 +711,9 @@ extern void bio_integrity_init(void); #else /* CONFIG_BLK_DEV_INTEGRITY */ -static inline int bio_integrity(struct bio *bio) +static inline void *bio_integrity(struct bio *bio) { - return 0; + return NULL; } static inline bool bio_integrity_enabled(struct bio *bio) @@ -754,6 +764,11 @@ static inline void bio_integrity_init(void) return; } +static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag) +{ + return false; +} + #endif /* CONFIG_BLK_DEV_INTEGRITY */ #endif /* CONFIG_BLOCK */ diff --git a/include/scsi/scsi_cmnd.h b/include/scsi/scsi_cmnd.h index 73f349044941..522a5f27f553 100644 --- a/include/scsi/scsi_cmnd.h +++ b/include/scsi/scsi_cmnd.h @@ -10,9 +10,10 @@ #include struct Scsi_Host; -struct scsi_device; struct scsi_driver; +#include + /* * MAX_COMMAND_SIZE is: * The longest fixed-length SCSI CDB as per the SCSI standard. @@ -81,6 +82,7 @@ struct scsi_cmnd { unsigned char prot_op; unsigned char prot_type; + unsigned char prot_flags; unsigned short cmd_len; enum dma_data_direction sc_data_direction; @@ -252,6 +254,14 @@ static inline unsigned char scsi_get_prot_op(struct scsi_cmnd *scmd) return scmd->prot_op; } +enum scsi_prot_flags { + SCSI_PROT_TRANSFER_PI = 1 << 0, + SCSI_PROT_GUARD_CHECK = 1 << 1, + SCSI_PROT_REF_CHECK = 1 << 2, + SCSI_PROT_REF_INCREMENT = 1 << 3, + SCSI_PROT_IP_CHECKSUM = 1 << 4, +}; + /* * The controller usually does not know anything about the target it * is communicating with. However, when DIX is enabled the controller @@ -280,6 +290,17 @@ static inline sector_t scsi_get_lba(struct scsi_cmnd *scmd) return blk_rq_pos(scmd->request); } +static inline unsigned int scsi_prot_interval(struct scsi_cmnd *scmd) +{ + return scmd->device->sector_size; +} + +static inline u32 scsi_prot_ref_tag(struct scsi_cmnd *scmd) +{ + return blk_rq_pos(scmd->request) >> + (ilog2(scsi_prot_interval(scmd)) - 9) & 0xffffffff; +} + static inline unsigned scsi_prot_sg_count(struct scsi_cmnd *cmd) { return cmd->prot_sdb ? cmd->prot_sdb->table.nents : 0; @@ -316,17 +337,12 @@ static inline void set_driver_byte(struct scsi_cmnd *cmd, char status) static inline unsigned scsi_transfer_length(struct scsi_cmnd *scmd) { unsigned int xfer_len = scsi_out(scmd)->length; - unsigned int prot_op = scsi_get_prot_op(scmd); - unsigned int sector_size = scmd->device->sector_size; + unsigned int prot_interval = scsi_prot_interval(scmd); - switch (prot_op) { - case SCSI_PROT_NORMAL: - case SCSI_PROT_WRITE_STRIP: - case SCSI_PROT_READ_INSERT: - return xfer_len; - } + if (scmd->prot_flags & SCSI_PROT_TRANSFER_PI) + xfer_len += (xfer_len >> ilog2(prot_interval)) * 8; - return xfer_len + (xfer_len >> ilog2(sector_size)) * 8; + return xfer_len; } #endif /* _SCSI_SCSI_CMND_H */ -- cgit v1.2.3